Merge branch 'master' into sexdeterrmine

modules/cnvkit/antitarget/main.nf

@@ -0,0 +1,36 @@
+process CNVKIT_ANTITARGET {
+    tag "$meta.id"
+    label 'process_low'
+
+    conda (params.enable_conda ? "bioconda::cnvkit=0.9.9" : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/cnvkit:0.9.9--pyhdfd78af_0':
+        'quay.io/biocontainers/cnvkit:0.9.9--pyhdfd78af_0' }"
+
+    input:
+    tuple val(meta), path(targets)
+
+    output:
+    tuple val(meta), path("*.bed"), emit: bed
+    path "versions.yml"           , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+
+    """
+    cnvkit.py \\
+        antitarget \\
+        $targets \\
+        --output ${prefix}.antitarget.bed \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        cnvkit: \$(cnvkit.py version | sed -e "s/cnvkit v//g")
+    END_VERSIONS
+    """
+}

modules/cnvkit/antitarget/meta.yml

@@ -0,0 +1,44 @@
+name: cnvkit_antitarget
+description:
+keywords:
+  - cvnkit
+  - antitarget
+tools:
+  - cnvkit:
+      description: |
+        CNVkit is a Python library and command-line software toolkit to infer and visualize copy number from high-throughput DNA sequencing data.
+        It is designed for use with hybrid capture, including both whole-exome and custom target panels, and short-read sequencing platforms such as Illumina and Ion Torrent.
+      homepage: https://cnvkit.readthedocs.io/en/stable/index.html
+      documentation: https://cnvkit.readthedocs.io/en/stable/index.html
+      tool_dev_url: "https://github.com/etal/cnvkit"
+      doi: 10.1371/journal.pcbi.1004873
+      licence: ["Apache-2.0"]
+
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - targets:
+      type: file
+      description: File containing genomic regions
+      pattern: "*.{bed}"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - bed:
+      type: file
+      description: File containing off-target regions
+      pattern: "*.{bed}"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@SusiJo"

modules/cnvkit/batch/main.nf

@@ -2,10 +2,10 @@ process CNVKIT_BATCH {
     tag "$meta.id"
     label 'process_low'
 
-    conda (params.enable_conda ? 'bioconda::cnvkit=0.9.9' : null)
+    conda (params.enable_conda ? 'bioconda::cnvkit=0.9.9 bioconda::samtools=1.15.1' : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/cnvkit:0.9.9--pyhdfd78af_0' :
+        'https://depot.galaxyproject.org/singularity/mulled-v2-780d630a9bb6a0ff2e7b6f730906fd703e40e98f:304d1c5ab610f216e77c61420ebe85f1e7c5968a-0' :
-        'quay.io/biocontainers/cnvkit:0.9.9--pyhdfd78af_0' }"
+        'quay.io/biocontainers/mulled-v2-780d630a9bb6a0ff2e7b6f730906fd703e40e98f:304d1c5ab610f216e77c61420ebe85f1e7c5968a-0' }"
 
     input:
     tuple val(meta), path(tumor), path(normal)
@@ -18,6 +18,8 @@ process CNVKIT_BATCH {
     tuple val(meta), path("*.cnn"), emit: cnn, optional: true
     tuple val(meta), path("*.cnr"), emit: cnr, optional: true
     tuple val(meta), path("*.cns"), emit: cns, optional: true
+    tuple val(meta), path("*.pdf"), emit: pdf, optional: true
+    tuple val(meta), path("*.png"), emit: png, optional: true
     path "versions.yml"           , emit: versions
 
     when:
@@ -25,21 +27,39 @@ process CNVKIT_BATCH {
 
     script:
     def args = task.ext.args ?: ''
-    def normal_args = normal ? "--normal $normal" : ""
+
-    def fasta_args = fasta ? "--fasta $fasta" : ""
+    // execute samtools only when cram files are input, cnvkit runs natively on bam but is prohibitively slow
+    // input pair is assumed to have same extension if both exist
+    def is_cram = tumor.Extension == "cram" ? true : false
+    def tumor_out = is_cram ? tumor.BaseName + ".bam" : "${tumor}"
+
+    // do not run samtools on normal samples in tumor_only mode
+    def normal_exists = normal ? true: false
+    // tumor_only mode does not need fasta & target
+    // instead it requires a pre-computed reference.cnn which is built from fasta & target
+    def (normal_out, normal_args, fasta_args) = ["", "", ""]
+
+    if (normal_exists){
+        def normal_prefix = normal.BaseName
+        normal_out = is_cram ? "${normal_prefix}" + ".bam" : "${normal}"
+        normal_args = normal_prefix ? "--normal $normal_out" : ""
+        fasta_args = fasta ? "--fasta $fasta" : ""
+    }
+
+    def target_args = targets ? "--targets $targets" : ""
     def reference_args = reference ? "--reference $reference" : ""
 
-    def target_args = ""
-    if (args.contains("--method wgs") || args.contains("-m wgs")) {
-        target_args = targets ? "--targets $targets" : ""
-    }
-    else {
-        target_args = "--targets $targets"
-    }
     """
+    if $is_cram; then
+        samtools view -T $fasta $tumor -@ $task.cpus -o $tumor_out
+        if $normal_exists; then
+            samtools view -T $fasta $normal -@ $task.cpus -o $normal_out
+        fi
+    fi
+
     cnvkit.py \\
         batch \\
-        $tumor \\
+        $tumor_out \\
         $normal_args \\
         $fasta_args \\
         $reference_args \\

modules/cnvkit/batch/meta.yml

@@ -11,27 +11,6 @@ tools:
       homepage: https://cnvkit.readthedocs.io/en/stable/index.html
       documentation: https://cnvkit.readthedocs.io/en/stable/index.html
       licence: ["Apache-2.0"]
-params:
-  - outdir:
-      type: string
-      description: |
-        The pipeline's output directory. By default, the module will
-        output files into `$params.outdir/<SOFTWARE>`
-  - publish_dir_mode:
-      type: string
-      description: |
-        Value for the Nextflow `publishDir` mode parameter.
-        Available: symlink, rellink, link, copy, copyNoFollow, move.
-  - enable_conda:
-      type: boolean
-      description: |
-        Run the module with Conda using the software specified
-        via the `conda` directive
-  - singularity_pull_docker_container:
-      type: boolean
-      description: |
-        Instead of directly downloading Singularity images for use with Singularity,
-        force the workflow to pull and convert Docker containers instead.
 input:
   - meta:
       type: map
@@ -49,7 +28,7 @@ input:
   - fasta:
       type: file
       description: |
-        Input reference genome fasta file
+        Input reference genome fasta file (only needed for cram_input and/or when normal_samples are provided)
   - targetfile:
       type: file
       description: |
@@ -80,6 +59,14 @@ output:
       type: file
       description: File containing copy number segment information
       pattern: "*.{cns}"
+  - pdf:
+      type: file
+      description: File with plot of copy numbers or segments on chromosomes
+      pattern: "*.{pdf}"
+  - png:
+      type: file
+      description: File with plot of bin-level log2 coverages and segmentation calls
+      pattern: "*.{png}"
   - versions:
       type: file
       description: File containing software versions
@@ -91,3 +78,4 @@ authors:
   - "@drpatelh"
   - "@fbdtemme"
   - "@lassefolkersen"
+  - "@SusiJo"

modules/cnvkit/reference/main.nf

@@ -0,0 +1,39 @@
+process CNVKIT_REFERENCE {
+    tag "$reference"
+    label 'process_low'
+
+    conda (params.enable_conda ? "bioconda::cnvkit=0.9.9" : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/cnvkit:0.9.9--pyhdfd78af_0':
+        'quay.io/biocontainers/cnvkit:0.9.9--pyhdfd78af_0' }"
+
+    input:
+    path    fasta
+    path    targets
+    path    antitargets
+
+    output:
+    path("*.cnn")                 , emit: cnn
+    path "versions.yml"           , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+
+    """
+    cnvkit.py \\
+        reference \\
+        --fasta $fasta \\
+        --targets $targets \\
+        --antitargets $antitargets \\
+        --output reference.cnn \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        cnvkit: \$(cnvkit.py version | sed -e "s/cnvkit v//g")
+    END_VERSIONS
+    """
+}

modules/cnvkit/reference/meta.yml

@@ -0,0 +1,47 @@
+name: cnvkit_reference
+description:
+keywords:
+  - cnvkit
+  - reference
+tools:
+  - cnvkit:
+      description: |
+        CNVkit is a Python library and command-line software toolkit to infer and visualize copy number from high-throughput DNA sequencing data.
+        It is designed for use with hybrid capture, including both whole-exome and custom target panels, and short-read sequencing platforms such as Illumina and Ion Torrent.
+      homepage: https://cnvkit.readthedocs.io/en/stable/index.html
+      documentation: https://cnvkit.readthedocs.io/en/stable/index.html
+      tool_dev_url: https://github.com/etal/cnvkit
+      doi: 10.1371/journal.pcbi.1004873
+      licence: ["Apache-2.0"]
+
+input:
+  - fasta:
+      type: file
+      description: File containing reference genome
+      pattern: "*.{fasta}"
+  - targets:
+      type: file
+      description: File containing genomic regions
+      pattern: "*.{bed}"
+  - antitargets:
+      type: file
+      description: File containing off-target genomic regions
+      pattern: "*.{bed}"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - reference:
+      type: file
+      description: File containing a copy-number reference (required for CNV calling in tumor_only mode)
+      pattern: "*.{cnn}"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@SusiJo"

modules/picard/collecthsmetrics/main.nf

@@ -24,7 +24,7 @@ process PICARD_COLLECTHSMETRICS {
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def reference = fasta ? "-R $fasta" : ""
+    def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : ""
 
     def avail_mem = 3
     if (!task.memory) {

modules/picard/collectmultiplemetrics/main.nf

@@ -22,6 +22,7 @@ process PICARD_COLLECTMULTIPLEMETRICS {
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
+    def reference = fasta ? "--REFERENCE_SEQUENCE ${fasta}" : ""
     def avail_mem = 3
     if (!task.memory) {
         log.info '[Picard CollectMultipleMetrics] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.'
@@ -35,7 +36,7 @@ process PICARD_COLLECTMULTIPLEMETRICS {
         $args \\
         --INPUT $bam \\
         --OUTPUT ${prefix}.CollectMultipleMetrics \\
-        --REFERENCE_SEQUENCE $fasta
+        $reference
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/picard/collectwgsmetrics/main.nf

@@ -34,7 +34,7 @@ process PICARD_COLLECTWGSMETRICS {
         $args \\
         --INPUT $bam \\
         --OUTPUT ${prefix}.CollectWgsMetrics.coverage_metrics \\
-        --REFERENCE_SEQUENCE $fasta
+        --REFERENCE_SEQUENCE ${fasta}
 
 
     cat <<-END_VERSIONS > versions.yml

modules/umitools/dedup/main.nf

@@ -9,12 +9,13 @@ process UMITOOLS_DEDUP {
 
     input:
     tuple val(meta), path(bam), path(bai)
+    val get_output_stats
 
     output:
     tuple val(meta), path("*.bam")             , emit: bam
-    tuple val(meta), path("*edit_distance.tsv"), emit: tsv_edit_distance
+    tuple val(meta), path("*edit_distance.tsv"), optional:true, emit: tsv_edit_distance
-    tuple val(meta), path("*per_umi.tsv")      , emit: tsv_per_umi
+    tuple val(meta), path("*per_umi.tsv")      , optional:true, emit: tsv_per_umi
-    tuple val(meta), path("*per_position.tsv") , emit: tsv_umi_per_position
+    tuple val(meta), path("*per_position.tsv") , optional:true, emit: tsv_umi_per_position
     path  "versions.yml"                       , emit: versions
 
     when:
@@ -24,12 +25,13 @@ process UMITOOLS_DEDUP {
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
     def paired = meta.single_end ? "" : "--paired"
+    def stats = get_output_stats ? "--output-stats $prefix" : ""
     """
     umi_tools \\
         dedup \\
         -I $bam \\
         -S ${prefix}.bam \\
-        --output-stats $prefix \\
+        $stats \\
         $paired \\
         $args
 

modules/umitools/dedup/meta.yml

@@ -26,6 +26,10 @@ input:
     description: |
       BAM index files corresponding to the input BAM file.
     pattern: "*.{bai}"
+  - get_output_stats:
+    type: boolean
+    description: |
+      Whether or not to generate output stats.
 output:
   - meta:
     type: map

tests/config/pytest_modules.yml

@@ -447,10 +447,18 @@ cmseq/polymut:
   - modules/cmseq/polymut/**
   - tests/modules/cmseq/polymut/**
 
+cnvkit/antitarget:
+  - modules/cnvkit/antitarget/**
+  - tests/modules/cnvkit/antitarget/**
+
 cnvkit/batch:
   - modules/cnvkit/batch/**
   - tests/modules/cnvkit/batch/**
 
+cnvkit/reference:
+  - modules/cnvkit/reference/**
+  - tests/modules/cnvkit/reference/**
+
 controlfreec/assesssignificance:
   - modules/controlfreec/assesssignificance/**
   - tests/modules/controlfreec/assesssignificance/**

tests/config/test_data.config

@@ -142,6 +142,7 @@ params {
                 genome_21_sizes                                = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/genome.sizes"
                 genome_21_interval_list                        = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/genome.interval_list"
                 genome_21_multi_interval_bed                   = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed"
+                genome_21_multi_interval_antitarget_bed        = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.antitarget.bed"
                 genome_21_multi_interval_bed_gz                = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz"
                 genome_21_multi_interval_bed_gz_tbi            = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/multi_intervals.bed.gz.tbi"
                 genome_21_chromosomes_dir                      = "${test_data_dir}/genomics/homo_sapiens/genome/chr21/sequence/chromosomes.tar.gz"

tests/modules/cnvkit/antitarget/main.nf

@@ -0,0 +1,16 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { CNVKIT_ANTITARGET } from '../../../../modules/cnvkit/antitarget/main.nf'
+
+workflow test_cnvkit_antitarget {
+
+    input = [
+        [ id:'test' ], // meta map
+        file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true)
+    ]
+
+    CNVKIT_ANTITARGET ( input )
+}
+

tests/modules/cnvkit/antitarget/nextflow.config

@@ -0,0 +1,5 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+    
+}

tests/modules/cnvkit/antitarget/test.yml

@@ -0,0 +1,8 @@
+- name: cnvkit antitarget test_cnvkit_antitarget
+  command: nextflow run ./tests/modules/cnvkit/antitarget -entry test_cnvkit_antitarget -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/antitarget/nextflow.config
+  tags:
+    - cnvkit
+    - cnvkit/antitarget
+  files:
+    - path: output/cnvkit/test.antitarget.bed
+      md5sum: 3d4d20f9f23b39970865d29ef239d20b

tests/modules/cnvkit/batch/main.nf

@@ -35,8 +35,8 @@ workflow test_cnvkit_cram {
 
     input = [
         [ id:'test'], // meta map
-        file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam'], checkIfExists: true),
+        file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_cram'], checkIfExists: true),
-        file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true)
+        file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_cram'], checkIfExists: true)
     ]
     fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
 
@@ -50,8 +50,20 @@ workflow test_cnvkit_tumoronly {
         file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_bam'], checkIfExists: true),
         []
     ]
-    fasta     = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
     reference = file(params.test_data['generic']['cnn']['reference'], checkIfExists: true)
 
     CNVKIT_TUMORONLY ( input, [], [], reference )
 }
+
+workflow test_cnvkit_tumoronly_cram {
+
+    input = [
+        [ id:'test'], // meta map
+        file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_sorted_cram'], checkIfExists: true),
+        []
+    ]
+    fasta     = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
+    reference = file(params.test_data['generic']['cnn']['reference'], checkIfExists: true)
+
+    CNVKIT_TUMORONLY ( input, fasta, [], reference )
+}

tests/modules/cnvkit/batch/test.yml

@@ -1,15 +1,14 @@
 - name: cnvkit batch test_cnvkit_hybrid
-  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_hybrid -c ./tests/config/nextflow.config -c ./tests/modules/cnvkit/batch/nextflow.config
+  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_hybrid -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/batch/nextflow.config
   tags:
-    - cnvkit/batch
     - cnvkit
+    - cnvkit/batch
   files:
     - path: output/cnvkit/baits.antitarget.bed
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: output/cnvkit/baits.target.bed
       md5sum: 26d25ff2d6c45b6d92169b3559c6acdb
     - path: output/cnvkit/reference.cnn
-      md5sum: ac99c1ad8b917b96ae15119146c91ab9
+      md5sum: 035d031f54c5f1b43b903da96559b475
     - path: output/cnvkit/test.paired_end.sorted.antitargetcoverage.cnn
       md5sum: 203caf8cef6935bb50b4138097955cb8
     - path: output/cnvkit/test.paired_end.sorted.bintest.cns
@@ -28,19 +27,18 @@
       md5sum: aa8a018b1d4d1e688c9f9f6ae01bf4d7
 
 - name: cnvkit batch test_cnvkit_wgs
-  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_wgs -c ./tests/config/nextflow.config -c ./tests/modules/cnvkit/batch/nextflow.config
+  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_wgs -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/batch/nextflow.config
   tags:
-    - cnvkit/batch
     - cnvkit
+    - cnvkit/batch
   files:
     - path: output/cnvkit/genome.antitarget.bed
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: output/cnvkit/genome.bed
       md5sum: 87a15eb9c2ff20ccd5cd8735a28708f7
     - path: output/cnvkit/genome.target.bed
       md5sum: a13353ae9c8405e701390c069255bbd2
     - path: output/cnvkit/reference.cnn
-      md5sum: 05c6211e0179885b8a83e44fd21d5f86
+      md5sum: 1606a85410bfaa79464be6e98699aa83
     - path: output/cnvkit/test.paired_end.sorted.antitargetcoverage.cnn
       md5sum: 203caf8cef6935bb50b4138097955cb8
     - path: output/cnvkit/test.paired_end.sorted.targetcoverage.cnn
@@ -59,19 +57,18 @@
       md5sum: 6ae6b3fce7299eedca6133d911c38fe1
 
 - name: cnvkit batch test_cnvkit_cram
-  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_cram -c ./tests/config/nextflow.config -c ./tests/modules/cnvkit/batch/nextflow.config
+  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_cram -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/batch/nextflow.config
   tags:
-    - cnvkit/batch
     - cnvkit
+    - cnvkit/batch
   files:
     - path: output/cnvkit/genome.antitarget.bed
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: output/cnvkit/genome.bed
       md5sum: 87a15eb9c2ff20ccd5cd8735a28708f7
     - path: output/cnvkit/genome.target.bed
       md5sum: a13353ae9c8405e701390c069255bbd2
     - path: output/cnvkit/reference.cnn
-      md5sum: 05c6211e0179885b8a83e44fd21d5f86
+      md5sum: 1606a85410bfaa79464be6e98699aa83
     - path: output/cnvkit/test.paired_end.sorted.antitargetcoverage.cnn
       md5sum: 203caf8cef6935bb50b4138097955cb8
     - path: output/cnvkit/test.paired_end.sorted.targetcoverage.cnn
@@ -90,12 +87,21 @@
       md5sum: 6ae6b3fce7299eedca6133d911c38fe1
 
 - name: cnvkit batch test_cnvkit_tumoronly
-  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_tumoronly -c ./tests/config/nextflow.config -c ./tests/modules/cnvkit/batch/nextflow.config
+  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_tumoronly -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/batch/nextflow.config
   tags:
-    - cnvkit/batch
     - cnvkit
+    - cnvkit/batch
+  files:
+    - path: output/cnvkit/reference.antitarget-tmp.bed
+    - path: output/cnvkit/reference.target-tmp.bed
+      md5sum: 26d25ff2d6c45b6d92169b3559c6acdb
+
+- name: cnvkit batch test_cnvkit_tumoronly_cram
+  command: nextflow run ./tests/modules/cnvkit/batch -entry test_cnvkit_tumoronly_cram -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/batch/nextflow.config
+  tags:
+    - cnvkit
+    - cnvkit/batch
   files:
     - path: output/cnvkit/reference.antitarget-tmp.bed
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: output/cnvkit/reference.target-tmp.bed
       md5sum: 26d25ff2d6c45b6d92169b3559c6acdb

tests/modules/cnvkit/reference/main.nf

@@ -0,0 +1,14 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { CNVKIT_REFERENCE } from '../../../../modules/cnvkit/reference/main.nf'
+
+workflow test_cnvkit_reference {
+
+    fasta       = file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true)
+    targets     = file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_bed'], checkIfExists: true)
+    antitargets = file(params.test_data['homo_sapiens']['genome']['genome_21_multi_interval_antitarget_bed'], checkIfExists: true)
+
+    CNVKIT_REFERENCE ( fasta, targets, antitargets )
+}

tests/modules/cnvkit/reference/nextflow.config

@@ -0,0 +1,5 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+    
+}

tests/modules/cnvkit/reference/test.yml

@@ -0,0 +1,8 @@
+- name: cnvkit reference test_cnvkit_reference
+  command: nextflow run ./tests/modules/cnvkit/reference -entry test_cnvkit_reference -c ./tests/config/nextflow.config  -c ./tests/modules/cnvkit/reference/nextflow.config
+  tags:
+    - cnvkit/reference
+    - cnvkit
+  files:
+    - path: output/cnvkit/reference.cnn
+      md5sum: 7c4a7902f5ab101b1f9d6038d331b3d9

tests/modules/picard/collecthsmetrics/main.nf

@@ -16,3 +16,14 @@ workflow test_picard_collecthsmetrics {
 
     PICARD_COLLECTHSMETRICS ( input, fasta, fai, bait_intervals, target_intervals )
 }
+
+workflow test_picard_collecthsmetrics_nofasta {
+
+    input = [ [ id:'test', single_end:false ], // meta map
+            file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ]
+
+    bait_intervals = file(params.test_data['sarscov2']['genome']['baits_interval_list'], checkIfExists: true)
+    target_intervals = file(params.test_data['sarscov2']['genome']['targets_interval_list'], checkIfExists: true)
+
+    PICARD_COLLECTHSMETRICS ( input, [], [], bait_intervals, target_intervals )
+}

tests/modules/picard/collecthsmetrics/test.yml

@@ -1,8 +1,15 @@
 - name: picard collecthsmetrics test_picard_collecthsmetrics
-  command: nextflow run ./tests/modules/picard/collecthsmetrics -entry test_picard_collecthsmetrics -c ./tests/config/nextflow.config -c ./tests/modules/picard/collecthsmetrics/nextflow.config
+  command: nextflow run ./tests/modules/picard/collecthsmetrics -entry test_picard_collecthsmetrics -c ./tests/config/nextflow.config  -c ./tests/modules/picard/collecthsmetrics/nextflow.config
   tags:
-    - picard
     - picard/collecthsmetrics
+    - picard
+  files:
+    - path: output/picard/test.CollectHsMetrics.coverage_metrics
+
+- name: picard collecthsmetrics test_picard_collecthsmetrics_nofasta
+  command: nextflow run ./tests/modules/picard/collecthsmetrics -entry test_picard_collecthsmetrics_nofasta -c ./tests/config/nextflow.config  -c ./tests/modules/picard/collecthsmetrics/nextflow.config
+  tags:
+    - picard/collecthsmetrics
+    - picard
   files:
-    # The file can't be md5'd consistently
     - path: output/picard/test.CollectHsMetrics.coverage_metrics

tests/modules/picard/collectmultiplemetrics/main.nf

@@ -5,10 +5,20 @@ nextflow.enable.dsl = 2
 include { PICARD_COLLECTMULTIPLEMETRICS } from '../../../../modules/picard/collectmultiplemetrics/main.nf'
 
 workflow test_picard_collectmultiplemetrics {
-    input = [ [ id:'test', single_end:false ], // meta map
+    input = [
-              file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true)
+                [ id:'test', single_end:false ], // meta map
+                file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true)
             ]
     fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
 
     PICARD_COLLECTMULTIPLEMETRICS ( input, fasta )
 }
+
+workflow test_picard_collectmultiplemetrics_nofasta {
+    input = [
+                [ id:'test', single_end:false ], // meta map
+                file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true)
+            ]
+
+    PICARD_COLLECTMULTIPLEMETRICS ( input, [] )
+}

tests/modules/picard/collectmultiplemetrics/test.yml

@@ -1,17 +1,33 @@
-- name: picard collectmultiplemetrics
+- name: picard collectmultiplemetrics test_picard_collectmultiplemetrics
-  command: nextflow run ./tests/modules/picard/collectmultiplemetrics -entry test_picard_collectmultiplemetrics -c ./tests/config/nextflow.config -c ./tests/modules/picard/collectmultiplemetrics/nextflow.config
+  command: nextflow run ./tests/modules/picard/collectmultiplemetrics -entry test_picard_collectmultiplemetrics -c ./tests/config/nextflow.config  -c ./tests/modules/picard/collectmultiplemetrics/nextflow.config
   tags:
     - picard
     - picard/collectmultiplemetrics
   files:
-    # These can't be md5'd consistently
+    - path: output/picard/test.CollectMultipleMetrics.alignment_summary_metrics
-    - path: ./output/picard/test.CollectMultipleMetrics.alignment_summary_metrics
+    - path: output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle.pdf
-    - path: ./output/picard/test.CollectMultipleMetrics.insert_size_metrics
+    - path: output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle_metrics
-    - path: ./output/picard/test.CollectMultipleMetrics.quality_distribution_metrics
+    - path: output/picard/test.CollectMultipleMetrics.insert_size_histogram.pdf
-    - path: ./output/picard/test.CollectMultipleMetrics.quality_by_cycle_metrics
+    - path: output/picard/test.CollectMultipleMetrics.insert_size_metrics
-    - path: ./output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle_metrics
+    - path: output/picard/test.CollectMultipleMetrics.quality_by_cycle.pdf
-    - path: ./output/picard/test.CollectMultipleMetrics.quality_by_cycle.pdf
+    - path: output/picard/test.CollectMultipleMetrics.quality_by_cycle_metrics
-    - path: ./output/picard/test.CollectMultipleMetrics.quality_distribution.pdf
+    - path: output/picard/test.CollectMultipleMetrics.quality_distribution.pdf
-    - path: ./output/picard/test.CollectMultipleMetrics.read_length_histogram.pdf
+    - path: output/picard/test.CollectMultipleMetrics.quality_distribution_metrics
-    - path: ./output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle.pdf
+    - path: output/picard/test.CollectMultipleMetrics.read_length_histogram.pdf
-    - path: ./output/picard/test.CollectMultipleMetrics.insert_size_histogram.pdf
+
+- name: picard collectmultiplemetrics test_picard_collectmultiplemetrics_nofasta
+  command: nextflow run ./tests/modules/picard/collectmultiplemetrics -entry test_picard_collectmultiplemetrics_nofasta -c ./tests/config/nextflow.config  -c ./tests/modules/picard/collectmultiplemetrics/nextflow.config
+  tags:
+    - picard
+    - picard/collectmultiplemetrics
+  files:
+    - path: output/picard/test.CollectMultipleMetrics.alignment_summary_metrics
+    - path: output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle.pdf
+    - path: output/picard/test.CollectMultipleMetrics.base_distribution_by_cycle_metrics
+    - path: output/picard/test.CollectMultipleMetrics.insert_size_histogram.pdf
+    - path: output/picard/test.CollectMultipleMetrics.insert_size_metrics
+    - path: output/picard/test.CollectMultipleMetrics.quality_by_cycle.pdf
+    - path: output/picard/test.CollectMultipleMetrics.quality_by_cycle_metrics
+    - path: output/picard/test.CollectMultipleMetrics.quality_distribution.pdf
+    - path: output/picard/test.CollectMultipleMetrics.quality_distribution_metrics
+    - path: output/picard/test.CollectMultipleMetrics.read_length_histogram.pdf

tests/modules/picard/collectwgsmetrics/test.yml

@@ -1,11 +1,7 @@
 - name: picard collectwgsmetrics test_picard_collectwgsmetrics
-  command: nextflow run ./tests/modules/picard/collectwgsmetrics -entry test_picard_collectwgsmetrics -c ./tests/config/nextflow.config -c ./tests/modules/picard/collectwgsmetrics/nextflow.config
+  command: nextflow run ./tests/modules/picard/collectwgsmetrics -entry test_picard_collectwgsmetrics -c ./tests/config/nextflow.config  -c ./tests/modules/picard/collectwgsmetrics/nextflow.config
   tags:
     - picard/collectwgsmetrics
     - picard
   files:
     - path: output/picard/test.CollectWgsMetrics.coverage_metrics
-      contains:
-        - "GENOME_TERRITORY"
-        - "29829"
-        - "17554"

tests/modules/umitools/dedup/main.nf

@@ -3,54 +3,81 @@
 nextflow.enable.dsl = 2
 
 include { UMITOOLS_EXTRACT } from '../../../../modules/umitools/extract/main.nf'
-include { BWA_INDEX } from '../../../../modules/bwa/index/main.nf'
+include { BWA_INDEX        } from '../../../../modules/bwa/index/main.nf'
-include { BWA_MEM   } from '../../../../modules/bwa/mem/main.nf'
+include { BWA_MEM          } from '../../../../modules/bwa/mem/main.nf'
 include { SAMTOOLS_INDEX   } from '../../../../modules/samtools/index/main.nf'
-include { UMITOOLS_DEDUP } from '../../../../modules/umitools/dedup/main.nf'
+include { UMITOOLS_DEDUP   } from '../../../../modules/umitools/dedup/main.nf'
 
 //
 // Test with no UMI
 //
 workflow test_umitools_dedup_no_umi {
-    input = [ [ id:'test'], // meta map
+    input = [ 
-              [ file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) ],
+        [ id:'test'], // meta map
-              [ file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) ]
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true),
-            ]
+        file(params.test_data['sarscov2']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true)
+    ]
+    get_output_stats = false
 
-    UMITOOLS_DEDUP ( input )
+    UMITOOLS_DEDUP ( input, get_output_stats )
 }
 
 //
-// Test with single-end data
+// Test with single-end data without --output-stats
 //
-workflow test_umitools_dedup_single_end {
+workflow test_umitools_dedup_single_end_no_stats {
-    input = [ [ id:'test', single_end:true ], // meta map
+    input = [ 
-              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ]
+        [ id:'test', single_end:true ], // meta map
-            ]
+        file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)
-
+    ]
     fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    get_output_stats = false
 
     UMITOOLS_EXTRACT ( input )
     BWA_INDEX ( fasta )
     BWA_MEM ( UMITOOLS_EXTRACT.out.reads, BWA_INDEX.out.index, true )
-    SAMTOOLS_INDEX (BWA_MEM.out.bam)
+    SAMTOOLS_INDEX ( BWA_MEM.out.bam )
-    UMITOOLS_DEDUP(BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]))
+    UMITOOLS_DEDUP ( BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]), get_output_stats )
 }
 
 //
-// Test with paired-end data
+// Test with paired-end data without --output-stats
 //
-workflow test_umitools_dedup_paired_end {
+workflow test_umitools_dedup_paired_end_no_stats {
-    input = [ [ id:'test', single_end:false ], // meta map
+    input = [ 
-              [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+        [ id:'test', single_end:false ], // meta map
-                file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ]
+        [ 
-            ]
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
-
+            file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) 
+        ]
+    ]
     fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    get_output_stats = false
 
     UMITOOLS_EXTRACT ( input )
     BWA_INDEX ( fasta )
     BWA_MEM ( UMITOOLS_EXTRACT.out.reads, BWA_INDEX.out.index, true )
-    SAMTOOLS_INDEX (BWA_MEM.out.bam)
+    SAMTOOLS_INDEX ( BWA_MEM.out.bam )
-    UMITOOLS_DEDUP(BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]))
+    UMITOOLS_DEDUP ( BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]), get_output_stats )
+}
+
+//
+// Test with paired-end data with --output-stats
+//
+workflow test_umitools_dedup_paired_end_stats {
+    input = [ 
+        [ id:'test', single_end:false ], // meta map
+        [ 
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+            file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) 
+        ]
+    ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    get_output_stats = true
+
+    UMITOOLS_EXTRACT ( input )
+    BWA_INDEX ( fasta )
+    BWA_MEM ( UMITOOLS_EXTRACT.out.reads, BWA_INDEX.out.index, true )
+    SAMTOOLS_INDEX ( BWA_MEM.out.bam )
+    UMITOOLS_DEDUP ( BWA_MEM.out.bam.join(SAMTOOLS_INDEX.out.bai, by: [0]), get_output_stats )
 }

tests/modules/umitools/dedup/nextflow.config

@@ -7,11 +7,7 @@ process {
     }
 
     withName: UMITOOLS_DEDUP {
-        ext.args = ''
+        ext.prefix = { "${meta.id}.dedup" }
-        ext.prefix = 'dedup'
     }
 
-    withName: BWA_MEM {
-        ext.args2 = ''
-    }
 }

tests/modules/umitools/dedup/test.yml

@@ -1,54 +1,87 @@
 - name: umitools dedup test_umitools_dedup_no_umi
-  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_no_umi -c tests/config/nextflow.config
+  command: nextflow run ./tests/modules/umitools/dedup -entry test_umitools_dedup_no_umi -c ./tests/config/nextflow.config  -c ./tests/modules/umitools/dedup/nextflow.config
   tags:
     - umitools/dedup
     - umitools
   files:
-    - path: output/umitools/dedup.bam
+    - path: output/umitools/test.dedup.bam
-      md5sum: 53b4edc399db81b87d2343e78af73cf0
-    - path: output/umitools/dedup_edit_distance.tsv
-      md5sum: 65186b0964e2f8d970cc04d736d8b119
-    - path: output/umitools/dedup_per_umi.tsv
-      md5sum: 8e6783a4a79437b095f095f2aefe7c01
-    - path: output/umitools/dedup_per_umi_per_position.tsv
-      md5sum: 9386db4a104b8e4e32f3ca4a84efa4ac
-    - path: output/umitools/versions.yml
-      md5sum: 4aaaa33565bcd9a984255139933d6446
 
-- name: umitools dedup test_umitools_dedup_single_end
+- name: umitools dedup test_umitools_dedup_single_end_no_stats
-  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_single_end -c tests/config/nextflow.config
+  command: nextflow run ./tests/modules/umitools/dedup -entry test_umitools_dedup_single_end_no_stats -c ./tests/config/nextflow.config  -c ./tests/modules/umitools/dedup/nextflow.config
   tags:
-    - umitools
     - umitools/dedup
+    - umitools
   files:
+    - path: output/bwa/bwa/genome.amb
+      md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e
+    - path: output/bwa/bwa/genome.ann
+      md5sum: c32e11f6c859f166c7525a9c1d583567
+    - path: output/bwa/bwa/genome.bwt
+      md5sum: 0469c30a1e239dd08f68afe66fde99da
+    - path: output/bwa/bwa/genome.pac
+      md5sum: 983e3d2cd6f36e2546e6d25a0da78d66
+    - path: output/bwa/bwa/genome.sa
+      md5sum: ab3952cabf026b48cd3eb5bccbb636d1
     - path: output/bwa/test.bam
-      md5sum: ea41a3cdca1856b22845e1067fd31f37
+      md5sum: 3ecbe569cadb9b6c881917ce60779f75
-    - path: output/bwa/versions.yml
-      md5sum: ce4d987f2c53f4c01b31d210c357b24a
     - path: output/samtools/test.bam.bai
       md5sum: 095af0ad3921212597ffd7c342ecd5a0
-    - path: output/samtools/versions.yml
+    - path: output/umitools/test.dedup.bam
-      md5sum: 69b7cde627c9b4e8403dfc125db71cc7
+    - path: output/umitools/test.umi_extract.fastq.gz
-    - path: output/umitools/dedup.bam
+    - path: output/umitools/test.umi_extract.log
-      md5sum: d95df177063432748ff33f473910cb1e
-    - path: output/umitools/versions.yml
-      md5sum: 730e768dd199d2f5bfb6fd0850446344
 
-- name: umitools dedup test_umitools_dedup_paired_end
+- name: umitools dedup test_umitools_dedup_paired_end_no_stats
-  command: nextflow run tests/modules/umitools/dedup -entry test_umitools_dedup_paired_end -c tests/config/nextflow.config
+  command: nextflow run ./tests/modules/umitools/dedup -entry test_umitools_dedup_paired_end_no_stats -c ./tests/config/nextflow.config  -c ./tests/modules/umitools/dedup/nextflow.config
   tags:
-    - umitools
     - umitools/dedup
+    - umitools
   files:
+    - path: output/bwa/bwa/genome.amb
+      md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e
+    - path: output/bwa/bwa/genome.ann
+      md5sum: c32e11f6c859f166c7525a9c1d583567
+    - path: output/bwa/bwa/genome.bwt
+      md5sum: 0469c30a1e239dd08f68afe66fde99da
+    - path: output/bwa/bwa/genome.pac
+      md5sum: 983e3d2cd6f36e2546e6d25a0da78d66
+    - path: output/bwa/bwa/genome.sa
+      md5sum: ab3952cabf026b48cd3eb5bccbb636d1
     - path: output/bwa/test.bam
-      md5sum: 1ad786cae0ff2254c655e3a206929617
+      md5sum: e7dcbac1825bf210409b762dbb4fec8f
-    - path: output/bwa/versions.yml
-      md5sum: b524c5ddf61c20f4a0a93ae8fc78b851
     - path: output/samtools/test.bam.bai
-      md5sum: 7496f4056a8e86327ca93e350f282fc2
+      md5sum: f75780d1de7860329b7fb4afeadc4bed
-    - path: output/samtools/versions.yml
+    - path: output/umitools/test.dedup.bam
-      md5sum: 72fc2ab934fd4bca0f7f14a705530d34
+    - path: output/umitools/test.umi_extract.log
-    - path: output/umitools/dedup.bam
+    - path: output/umitools/test.umi_extract_1.fastq.gz
-      md5sum: e8d1eae2aacef76254948c5568e94555
+    - path: output/umitools/test.umi_extract_2.fastq.gz
-    - path: output/umitools/versions.yml
+
-      md5sum: fd39e05042d354b3d8de49b617d3183d
+- name: umitools dedup test_umitools_dedup_paired_end_stats
+  command: nextflow run ./tests/modules/umitools/dedup -entry test_umitools_dedup_paired_end_stats -c ./tests/config/nextflow.config  -c ./tests/modules/umitools/dedup/nextflow.config
+  tags:
+    - umitools/dedup
+    - umitools
+  files:
+    - path: output/bwa/bwa/genome.amb
+      md5sum: 3a68b8b2287e07dd3f5f95f4344ba76e
+    - path: output/bwa/bwa/genome.ann
+      md5sum: c32e11f6c859f166c7525a9c1d583567
+    - path: output/bwa/bwa/genome.bwt
+      md5sum: 0469c30a1e239dd08f68afe66fde99da
+    - path: output/bwa/bwa/genome.pac
+      md5sum: 983e3d2cd6f36e2546e6d25a0da78d66
+    - path: output/bwa/bwa/genome.sa
+      md5sum: ab3952cabf026b48cd3eb5bccbb636d1
+    - path: output/bwa/test.bam
+      md5sum: e7dcbac1825bf210409b762dbb4fec8f
+    - path: output/samtools/test.bam.bai
+      md5sum: f75780d1de7860329b7fb4afeadc4bed
+    - path: output/umitools/test.dedup.bam
+    - path: output/umitools/test.dedup_edit_distance.tsv
+      md5sum: c247a49b58768e6e2e86a6c08483e612
+    - path: output/umitools/test.dedup_per_umi.tsv
+      md5sum: 10e35ca37f2bfb521ac6dd7314951a68
+    - path: output/umitools/test.dedup_per_umi_per_position.tsv
+      md5sum: 2e1a12e6f720510880068deddeefe063
+    - path: output/umitools/test.umi_extract.log
+    - path: output/umitools/test.umi_extract_1.fastq.gz
+    - path: output/umitools/test.umi_extract_2.fastq.gz