Add tests for umitools extract and dedup (#1429)

* NGSCheckMate v1

* Add some tests for UMItools

* Added tests for dedup

* Include pytest

* Delete main.nf

* Delete meta.yml

* Delete main.nf

* Delete nextflow.config

* Delete test.yml

* add prettier

* Add direct test on bam

* Update tests/modules/umitools/dedup/main.nf

Co-authored-by: Edmund Miller <edmund.a.miller@gmail.com>

* Update tests/modules/umitools/dedup/main.nf

Co-authored-by: Edmund Miller <edmund.a.miller@gmail.com>

* Update tests/config/pytest_modules.yml

Co-authored-by: Edmund Miller <edmund.a.miller@gmail.com>

* Not ignore-umi

Co-authored-by: Simon Pearce <simon.pearce@cruk.manchester.ac.uk>
Co-authored-by: Maxime U. Garcia <max.u.garcia@gmail.com>
Co-authored-by: Edmund Miller <edmund.a.miller@gmail.com>

tests/config/pytest_modules.yml

@@ -1763,6 +1763,15 @@ ultra/pipeline:
   - modules/ultra/pipeline/**
   - tests/modules/ultra/pipeline/**
 
+umitools/dedup:
+  - modules/umitools/dedup/**
+  - modules/umitools/extract/**
+  - tests/modules/umitools/dedup/**
+
+umitools/extract:
+  - modules/umitools/extract/**
+  - tests/modules/umitools/extract/**
+
 unicycler:
   - modules/unicycler/**
   - tests/modules/unicycler/**

tests/modules/umitools/dedup/main.nf

@@ -0,0 +1,56 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { UMITOOLS_EXTRACT } from '../../../../modules/umitools/extract/main.nf'
+include { BWA_INDEX } from '../../../../modules/bwa/index/main.nf'
+include { BWA_MEM   } from '../../../../modules/bwa/mem/main.nf'
+include { SAMTOOLS_INDEX   } from '../../../../modules/samtools/index/main.nf'
+include { UMITOOLS_DEDUP } from '../../../../modules/umitools/dedup/main.nf'
+
+//
+// Test with no UMI
+//
+workflow test_umitools_dedup_no_umi {
+    input = [ [ id:'test'], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) ],
+              [ file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) ]
+            ]
+
+    UMITOOLS_DEDUP ( input )
+}
+
+//
+// Test with single-end data
+//
+workflow test_umitools_dedup_single_end {
+    input = [ [ id:'test', single_end:true ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ]
+            ]
+
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+
+    UMITOOLS_EXTRACT ( input )
+    BWA_INDEX ( fasta )
+    BWA_MEM ( UMITOOLS_EXTRACT.out.reads, BWA_INDEX.out.index, true )
+    SAMTOOLS_INDEX (BWA_MEM.out.bam)
+    UMITOOLS_DEDUP(BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]))
+}
+
+//
+// Test with paired-end data
+//
+workflow test_umitools_dedup_paired_end {
+    input = [ [ id:'test', single_end:false ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+                file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ]
+            ]
+
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+
+    UMITOOLS_EXTRACT ( input )
+    BWA_INDEX ( fasta )
+    BWA_MEM ( UMITOOLS_EXTRACT.out.reads, BWA_INDEX.out.index, true )
+    SAMTOOLS_INDEX (BWA_MEM.out.bam)
+    UMITOOLS_DEDUP(BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]))
+}

tests/modules/umitools/dedup/nextflow.config

@@ -0,0 +1,17 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: UMITOOLS_EXTRACT {
+        ext.args = '--bc-pattern="NNNN"'
+    }
+
+    withName: UMITOOLS_DEDUP {
+        ext.args = ''
+        ext.prefix = 'dedup'
+    }
+
+    withName: BWA_MEM {
+        ext.args2 = ''
+    }
+}

tests/modules/umitools/dedup/test.yml

@@ -0,0 +1,54 @@
+- name: umitools dedup test_umitools_dedup_no_umi
+  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_no_umi -c tests/config/nextflow.config
+  tags:
+    - umitools/dedup
+    - umitools
+  files:
+    - path: output/umitools/dedup.bam
+      md5sum: 53b4edc399db81b87d2343e78af73cf0
+    - path: output/umitools/dedup_edit_distance.tsv
+      md5sum: 65186b0964e2f8d970cc04d736d8b119
+    - path: output/umitools/dedup_per_umi.tsv
+      md5sum: 8e6783a4a79437b095f095f2aefe7c01
+    - path: output/umitools/dedup_per_umi_per_position.tsv
+      md5sum: 9386db4a104b8e4e32f3ca4a84efa4ac
+    - path: output/umitools/versions.yml
+      md5sum: 4aaaa33565bcd9a984255139933d6446
+
+- name: umitools dedup test_umitools_dedup_single_end
+  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_single_end -c tests/config/nextflow.config
+  tags:
+    - umitools
+    - umitools/dedup
+  files:
+    - path: output/bwa/test.bam
+      md5sum: ea41a3cdca1856b22845e1067fd31f37
+    - path: output/bwa/versions.yml
+      md5sum: ce4d987f2c53f4c01b31d210c357b24a
+    - path: output/samtools/test.bam.bai
+      md5sum: 095af0ad3921212597ffd7c342ecd5a0
+    - path: output/samtools/versions.yml
+      md5sum: 69b7cde627c9b4e8403dfc125db71cc7
+    - path: output/umitools/dedup.bam
+      md5sum: d95df177063432748ff33f473910cb1e
+    - path: output/umitools/versions.yml
+      md5sum: 730e768dd199d2f5bfb6fd0850446344
+
+- name: umitools dedup test_umitools_dedup_paired_end
+  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_paired_end -c tests/config/nextflow.config
+  tags:
+    - umitools
+    - umitools/dedup
+  files:
+    - path: output/bwa/test.bam
+      md5sum: 1ad786cae0ff2254c655e3a206929617
+    - path: output/bwa/versions.yml
+      md5sum: b524c5ddf61c20f4a0a93ae8fc78b851
+    - path: output/samtools/test.bam.bai
+      md5sum: 7496f4056a8e86327ca93e350f282fc2
+    - path: output/samtools/versions.yml
+      md5sum: 72fc2ab934fd4bca0f7f14a705530d34
+    - path: output/umitools/dedup.bam
+      md5sum: e8d1eae2aacef76254948c5568e94555
+    - path: output/umitools/versions.yml
+      md5sum: fd39e05042d354b3d8de49b617d3183d

tests/modules/umitools/extract/main.nf

@@ -0,0 +1,29 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { UMITOOLS_EXTRACT } from '../../../../modules/umitools/extract/main.nf'
+
+//
+// Test with single-end data
+//
+workflow test_umitools_extract_single_end {
+    input = [ [ id:'test', single_end:true ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ]
+            ]
+
+    UMITOOLS_EXTRACT ( input )
+}
+
+//
+// Test with paired-end data
+//
+workflow test_umitools_extract_paired_end {
+    input = [ [ id:'test', single_end:false ], // meta map
+              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+                file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ]
+            ]
+
+    UMITOOLS_EXTRACT ( input )
+}
+

tests/modules/umitools/extract/nextflow.config

@@ -0,0 +1,9 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: UMITOOLS_EXTRACT {
+        ext.args = '--bc-pattern="NNNN"'
+    }
+
+}

tests/modules/umitools/extract/test.yml

@@ -0,0 +1,27 @@
+- name: umitools extract test_umitools_extract_single_end
+  command: nextflow run tests/modules/umitools/extract -entry test_umitools_extract_single_end -c tests/config/nextflow.config
+  tags:
+    - umitools/extract
+    - umitools
+  files:
+    - path: output/umitools/test.umi_extract.fastq.gz
+      should_exist: true
+    - path: output/umitools/test.umi_extract.log
+      contains: ["job finished in"]
+    - path: output/umitools/versions.yml
+      md5sum: 397e6972343f9d7b8eae387fc18c12c7
+
+- name: umitools extract test_umitools_extract_paired_end
+  command: nextflow run tests/modules/umitools/extract -entry test_umitools_extract_paired_end -c tests/config/nextflow.config
+  tags:
+    - umitools/extract
+    - umitools
+  files:
+    - path: output/umitools/test.umi_extract.log
+      contains: ["job finished in"]
+    - path: output/umitools/test.umi_extract_1.fastq.gz
+      should_exist: true
+    - path: output/umitools/test.umi_extract_2.fastq.gz
+      should_exist: true
+    - path: output/umitools/versions.yml
+      md5sum: 0aec6f919d62b7b79f6d0c5d79411464