Allow multiple Fastq files in rnafusion output

.vscode/settings.json

@@ -1,3 +1,3 @@
 {
-    "nextflow.telemetry.enabled": false
-}
+  "nextflow.telemetry.enabled": false
+}

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.7
+
+1. Rnafusion samplesheets will now contain multiple entries of the same sample when fastq files from multiple lanes are created by nf-cmgg/preprocessing
+
 ## 0.2.6
 
 1. Convert the fastq yield to a Long type instead of Integer to prevent integer overflows

build.gradle

@@ -6,7 +6,7 @@ dependencies {
     implementation 'com.squareup.okhttp3:okhttp:5.3.2'
 }
 
-version = '0.2.6'
+version = '0.2.7'
 
 nextflowPlugin {
     nextflowVersion = '25.10.0'

src/main/groovy/nfcmgg/plugin/samplesheets/OutputEntry.groovy

@@ -29,9 +29,9 @@ import groovy.util.logging.Slf4j
 @CompileDynamic
 class OutputEntry {
 
-    final private Map<String, String> values
+    final private Map<String, Object> values
 
-    OutputEntry(Map<String, String> defaultValues = [:]) {
+    OutputEntry(Map<String, Object> defaultValues = [:]) {
         this.values = defaultValues
     }
 
@@ -63,20 +63,32 @@ class OutputEntry {
         return newEntry
     }
 
-    OutputEntry add(String key, String value) {
+    OutputEntry append(String key, String value) {
+        if (!this.values.containsKey(key)) {
+            this.values[key] = []
+        }
+        this.values[key] << value
+        return this
+    }
+
+    OutputEntry add(String key, Object value) {
         this.values[key] = value
         return this
     }
 
-    String get(String key) {
+    String getAsString(String key) {
+        return this.values.get(key)?.toString()
+    }
+
+    Object get(String key) {
         return this.values.get(key)
     }
 
-    String get(String key, String defaultValue) {
+    Object get(String key, Object defaultValue) {
         return this.values.get(key, defaultValue)
     }
 
-    Map<String, String> getValues() {
+    Map<String, Object> getValues() {
         return this.values
     }
 

src/main/groovy/nfcmgg/plugin/samplesheets/PipelineObserver.groovy

@@ -78,7 +78,7 @@ class PipelineObserver implements TraceObserverV2 {
      * @return a map of key-value pairs to be added to the sample entry when it is first created
      */
     /* groovylint-disable-next-line UnusedMethodParameter */
-    Map getDefaultValuesForSample(String sample) {
+    Map<String, Object> getDefaultValuesForSample(String sample) {
         return [:]
     }
 
@@ -120,7 +120,11 @@ class PipelineObserver implements TraceObserverV2 {
                     "Multiple possible samples found for path '$basePath': $possibleSamples, using '$sample' as sample name, because it is the longest match"
                 )
         }
-        entries.putIfAbsent(sample, new OutputEntry(['id': sample] + getDefaultValuesForSample(sample)))
+        entries.putIfAbsent(
+            sample,
+            /* groovylint-disable-next-line UnnecessaryCast */
+            new OutputEntry(['id': sample] as Map<String, Object> + getDefaultValuesForSample(sample))
+        )
         return sample
     }
 

src/main/groovy/nfcmgg/plugin/samplesheets/PreprocessingObserver.groovy

@@ -19,6 +19,7 @@ import groovy.util.logging.Slf4j
 import groovy.transform.CompileStatic
 
 import java.nio.file.Path
+import java.util.regex.Matcher
 
 import nextflow.trace.event.FilePublishEvent
 
@@ -47,7 +48,11 @@ class PreprocessingObserver extends PipelineObserver {
                 sampleMetrics.each { metric ->
                     String sample = metric['Sample']
                     String yield = metric['yield_']
-                    entries.putIfAbsent(sample, new OutputEntry(['id': sample] + getDefaultValuesForSample(sample)))
+                    entries.putIfAbsent(
+                        sample,
+                        /* groovylint-disable-next-line UnnecessaryCast */
+                        new OutputEntry(['id': sample] as Map<String, Object> + getDefaultValuesForSample(sample))
+                    )
                     entries[sample].add('yield', yield)
                 }
             }
@@ -67,11 +72,11 @@ class PreprocessingObserver extends PipelineObserver {
             case ~/^.*\.crai$/:
                 entries[safeGetSample(targetName)].add('crai', targetPath)
                 break
-            case ~/^.*R1_00.\.fastq\.gz$/:
-                entries[safeGetSample(targetName)].add('fastq_1', targetPath)
+            case ~/^.*R1_\d\d\d\.fastq\.gz$/:
+                entries[safeGetSample(targetName)].append('fastq_1', targetPath)
                 break
-            case ~/^.*R2_00.\.fastq\.gz$/:
-                entries[safeGetSample(targetName)].add('fastq_2', targetPath)
+            case ~/^.*R2_\d\d\d\.fastq\.gz$/:
+                entries[safeGetSample(targetName)].append('fastq_2', targetPath)
                 break
             case ~/^.*\.per-base\.bed\.gz$/:
                 entries[safeGetSample(targetName)].add('per_base_bed', targetPath)
@@ -89,14 +94,14 @@ class PreprocessingObserver extends PipelineObserver {
         entries = entries.sort()
         Map<String, OutputEntry> humanEntries = entries.findAll { entry ->
             // Only retain samples of human data for the samplesheets
-            entry.value.get('organism')?.toLowerCase() == 'homo sapiens' ||
-            entry.value.get('genome')?.toLowerCase() == 'grch38'
+            entry.value.getAsString('organism')?.toLowerCase() == 'homo sapiens' ||
+            entry.value.getAsString('genome')?.toLowerCase() == 'grch38'
         }
 
         Map<String, OutputEntry> mouseEntries = entries.findAll { entry ->
-            // Only retain samples of human data for the samplesheets
-            entry.value.get('organism')?.toLowerCase() == 'mus musculus' ||
-            entry.value.get('genome')?.toLowerCase() == 'mm10'
+            // Only retain samples of mouse data for the samplesheets
+            entry.value.getAsString('organism')?.toLowerCase() == 'mus musculus' ||
+            entry.value.getAsString('genome')?.toLowerCase() == 'mm10'
         }
 
         //
@@ -106,8 +111,8 @@ class PreprocessingObserver extends PipelineObserver {
             humanEntries
                 .findAll { entry ->
                     // Only create samplesheet for WES and WGS runs of DNA samples
-                    entry.value.get('sample_type')?.toLowerCase() == 'dna' &&
-                    entry.value.get('tag')?.toLowerCase() in ['wes', 'wgs']
+                    entry.value.getAsString('sample_type')?.toLowerCase() == 'dna' &&
+                    entry.value.getAsString('tag')?.toLowerCase() in ['wes', 'wgs']
                 }
                 .values()
                 *.subKeys([
@@ -129,9 +134,9 @@ class PreprocessingObserver extends PipelineObserver {
         Map<String, OutputEntry> rnafusionEntries = humanEntries
             .findAll { entry ->
                 // Only create samplesheet for RNAseqMDG runs of RNA samples that have FASTQ output
-                entry.value.get('sample_type')?.toLowerCase() == 'rna' &&
-                entry.value.get('tag')?.toLowerCase() == 'rnaseqmdg' &&
-                entry.value.get('fastq_1')
+                entry.value.getAsString('sample_type')?.toLowerCase() == 'rna' &&
+                entry.value.getAsString('tag')?.toLowerCase() == 'rnaseqmdg' &&
+                entry.value.getAsString('fastq_1')
             }
         List rnafusionKeys = [
             ['id', 'sample'],
@@ -142,24 +147,78 @@ class PreprocessingObserver extends PipelineObserver {
         ]
 
         // Passed data
-        creator.dump(
-            rnafusionEntries
-                .findAll { entry ->
-                    entry.value.get('yield').toLong() >= 1000000L
+        List<OutputEntry> passedEntries = []
+        rnafusionEntries
+            .findAll { entry ->
+                entry.value.getAsString('yield').toLong() >= 1000000L
+            }
+            .values()
+            *.subKeys(rnafusionKeys)
+            .each { OutputEntry entry ->
+                Map<String, String> fastq2Lanes = (entry.get('fastq_2', []) as List<String>).collectEntries { fastq2 ->
+                    Matcher match = fastq2 =~ ~/^.*R2_(\d\d\d)\.fastq\.gz$/
+                    if (!match.find()) {
+                        log.warn("Could not find lane for fastq2 file '$fastq2', skipping this file")
+                        return
+                    }
+                    String lane = match.group(1)
+                    [lane, fastq2]
                 }
-                .values()
-                *.subKeys(rnafusionKeys),
+
+                (entry.get('fastq_1') as List<String>).sort().each { fastq1 ->
+                    Matcher match = fastq1 =~ ~/^.*R1_(\d\d\d)\.fastq\.gz$/
+                    if (!match.find()) {
+                        log.warn("Could not find lane for fastq1 file '$fastq1', skipping this file")
+                        return
+                    }
+                    String lane = match.group(1)
+                    String fastq2 = fastq2Lanes.get(lane, '')
+                    passedEntries << new OutputEntry(entry.values.clone() as Map<String,Object>)
+                        .add('fastq_1', fastq1)
+                        .add('fastq_2', fastq2)
+                }
+            }
+
+        creator.dump(
+            passedEntries,
             location.resolve('nfcore_rnafusion_samplesheet.yaml')
         )
 
         // Failed data
-        creator.dump(
-            rnafusionEntries
-                .findAll { entry ->
-                    entry.value.get('yield').toLong() < 1000000L
+        List<OutputEntry> failedEntries = []
+        rnafusionEntries
+            .findAll { entry ->
+                entry.value.getAsString('yield').toLong() < 1000000L
+            }
+            .values()
+            *.subKeys(rnafusionKeys)
+            .each { OutputEntry entry ->
+                Map<String, String> fastq2Lanes = (entry.get('fastq_2', []) as List<String>).collectEntries { fastq2 ->
+                    Matcher match = fastq2 =~ ~/^.*R2_(\d\d\d)\.fastq\.gz$/
+                    if (!match.find()) {
+                        log.warn("Could not find lane for fastq2 file '$fastq2', skipping this file")
+                        return
+                    }
+                    String lane = match.group(1)
+                    [lane, fastq2]
                 }
-                .values()
-                *.subKeys(rnafusionKeys),
+
+                (entry.get('fastq_1') as List<String>).sort().each { fastq1 ->
+                    Matcher match = fastq1 =~ ~/^.*R1_(\d\d\d)\.fastq\.gz$/
+                    if (!match.find()) {
+                        log.warn("Could not find lane for fastq1 file '$fastq1', skipping this file")
+                        return
+                    }
+                    String lane = match.group(1)
+                    String fastq2 = fastq2Lanes.get(lane, '')
+                    failedEntries << new OutputEntry(entry.values.clone() as Map<String,Object>)
+                        .add('fastq_1', fastq1)
+                        .add('fastq_2', fastq2)
+                }
+            }
+
+        creator.dump(
+            failedEntries,
             location.resolve('nfcore_rnafusion_samplesheet_failed.yaml')
         )
 
@@ -169,9 +228,9 @@ class PreprocessingObserver extends PipelineObserver {
         creator.dump(
             (humanEntries + mouseEntries)
                 .findAll { entry ->
-                    String type = entry.value.get('sample_type')?.toLowerCase()
+                    String type = entry.value.getAsString('sample_type')?.toLowerCase()
                     // Only create samplesheet for DNA and tissue samples that are not mitochondrial
-                    (type == 'dna' || type == 'tissue') && !entry.value.get('id')?.startsWith('mtD')
+                    (type == 'dna' || type == 'tissue') && !entry.value.getAsString('id')?.startsWith('mtD')
                 }
                 .values()
                 *.subKeys([
@@ -196,8 +255,8 @@ class PreprocessingObserver extends PipelineObserver {
             humanEntries
                 .findAll { entry ->
                     // Only create samplesheet for WES runs of DNA samples
-                    entry.value.get('sample_type')?.toLowerCase() == 'dna' &&
-                    entry.value.get('tag')?.toLowerCase() in ['wes']
+                    entry.value.getAsString('sample_type')?.toLowerCase() == 'dna' &&
+                    entry.value.getAsString('tag')?.toLowerCase() in ['wes']
                 }
                 .values()
                 *.subKeys([
@@ -219,8 +278,8 @@ class PreprocessingObserver extends PipelineObserver {
             humanEntries
                 .findAll { entry ->
                     // Only create samplesheet for DNA samples
-                    entry.value.get('sample_type')?.toLowerCase() == 'dna' &&
-                    entry.value.get('tag')?.toLowerCase() in ['wes', 'wgs']
+                    entry.value.getAsString('sample_type')?.toLowerCase() == 'dna' &&
+                    entry.value.getAsString('tag')?.toLowerCase() in ['wes', 'wgs']
                 }
                 .values()
                 *.subKeys([

tests/preprocessing_mock/input.yaml

@@ -45,6 +45,17 @@
   family_number: Proband_123456
   library: RNA_prep
   reads_to_use_in_test: 1200000
+## An RNA sample with multiple lanes (rnafusion) - passed
+- id: R56789
+  samplename: R56789
+  genome: GRCh38
+  tag: RNAseqMDG
+  aligner: false
+  sample_type: RNA
+  family_number: Proband_123456
+  library: RNA_prep
+  reads_to_use_in_test: 1200000
+  lanes: 3
 ## A standard RNA sample (rnafusion) - failed
 - id: R654321
   samplename: R654321
@@ -55,6 +66,17 @@
   family_number: Proband_123456
   library: RNA_prep
   reads_to_use_in_test: 150
+## An RNA sample with multiple lanes (rnafusion) - failed
+- id: R654322
+  samplename: R654322
+  genome: GRCh38
+  tag: RNAseqMDG
+  aligner: false
+  sample_type: RNA
+  family_number: Proband_123456
+  library: RNA_prep
+  reads_to_use_in_test: 150
+  lanes: 3
 ## Flowcell
 - id: flowcell
   sample_info: ../../../tests/preprocessing_mock/sampleinfo.yaml

tests/preprocessing_mock/main.nf

@@ -12,7 +12,6 @@ workflow {
             input.addAll(new groovy.yaml.YamlSlurper().parseText(file(entry.sample_info).text))
         }
     }
-    println input.findAll { entry -> entry.sample_type == 'RNA'}
     MOCK_OUTPUT(input)
 
     publish:
@@ -35,7 +34,15 @@ process MOCK_OUTPUT {
     script:
     def fastqs = input_list
         .findAll { entry -> entry.aligner == false }
-        .collect { entry -> "echo '' | gzip > ${entry.samplename}_R1_001.fastq.gz && echo '' | gzip > ${entry.samplename}_R2_001.fastq.gz"}
+        .collect { entry ->
+            def lanes = entry.lanes?.toInteger() ?: 1
+            def fastq_echos = (lanes > 1 ? 1..lanes : [1])
+                .collect { lane ->
+                    "echo '' | gzip > ${entry.samplename}_R1_00${lane}.fastq.gz && echo '' | gzip > ${entry.samplename}_R2_00${lane}.fastq.gz"
+                }
+            return fastq_echos
+        }
+        .flatten()
         .join("\n    ")
     def crams = input_list
         .findAll { entry -> entry.aligner instanceof String }