Merge branch 'master' into gatk_426

modules/bowtie2/align/main.nf

@@ -11,6 +11,7 @@ process BOWTIE2_ALIGN {
     tuple val(meta), path(reads)
     path  index
     val   save_unaligned
+    val   sort_bam
 
     output:
     tuple val(meta), path("*.bam")    , emit: bam
@@ -36,8 +37,7 @@ process BOWTIE2_ALIGN {
         reads_args = "-1 ${reads[0]} -2 ${reads[1]}"
     }
 
-    def samtools_command = "samtools view -@ $task.cpus --bam --with-header ${args2} > ${prefix}.bam"
-
+    def samtools_command = sort_bam ? 'sort' : 'view'
 
     """
     INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/.rev.1.bt2//"`
@@ -51,7 +51,7 @@ process BOWTIE2_ALIGN {
         $unaligned \\
         $args \\
         2> ${prefix}.bowtie2.log \\
-        | $samtools_command
+        | samtools $samtools_command $args2 --threads $task.cpus -o ${prefix}.bam -
 
     if [ -f ${prefix}.unmapped.fastq.1.gz ]; then
         mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz
@@ -69,4 +69,3 @@ process BOWTIE2_ALIGN {
     END_VERSIONS
     """
 }
-

modules/bowtie2/align/meta.yml

@@ -29,6 +29,15 @@ input:
       type: file
       description: Bowtie2 genome index files
       pattern: "*.ebwt"
+  - save_unaligned:
+      type: boolean
+      description: |
+        Save reads that do not map to the reference (true) or discard them (false)
+        (default: false)
+  - sort_bam:
+      type: boolean
+      description: use samtools sort (true) or samtools view (false)
+      pattern: "true or false"
 output:
   - bam:
       type: file

modules/gatk4/splitintervals/main.nf

@@ -0,0 +1,48 @@
+process GATK4_SPLITINTERVALS {
+    tag "$meta.id"
+    label 'process_low'
+
+    conda (params.enable_conda ? "bioconda::gatk4=4.2.6.1" : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/gatk4:4.2.6.1--hdfd78af_0':
+        'quay.io/biocontainers/gatk4:4.2.6.1--hdfd78af_0' }"
+
+    input:
+    tuple val(meta), path(intervals)
+    path(fasta)
+    path(fasta_fai)
+    path(dict)
+
+    output:
+    tuple val(meta), path("**.interval_list"), emit: split_intervals
+    path "versions.yml"                      , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    def reference = fasta ? "--reference $fasta" : ""
+
+    def avail_mem = 3
+    if (!task.memory) {
+        log.info '[GATK SplitIntervals] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.'
+    } else {
+        avail_mem = task.memory.giga
+    }
+
+    """
+    gatk --java-options "-Xmx${avail_mem}g" SplitIntervals \\
+        --output ${prefix} \\
+        --intervals $intervals \\
+        $reference \\
+        --tmp-dir . \\
+        $args
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//')
+    END_VERSIONS
+    """
+}

modules/gatk4/splitintervals/meta.yml

@@ -0,0 +1,53 @@
+name: gatk4_splitintervals
+keywords:
+  - interval
+  - bed
+tools:
+  - gatk4:
+      description: Genome Analysis Toolkit (GATK4)
+      homepage: https://gatk.broadinstitute.org/hc/en-us
+      documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s
+      tool_dev_url: https://github.com/broadinstitute/gatk
+      doi: "10.1158/1538-7445.AM2017-3590"
+      licence: ["BSD-3-clause"]
+
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - interval:
+      type: file
+      description: Interval list or BED
+      pattern: "*.{interval,interval_list,bed}"
+  - fasta:
+      type: file
+      description: Reference FASTA
+      pattern: "*.{fa,fasta}"
+  - fasta_fai:
+      type: file
+      description: Reference FASTA index
+      pattern: "*.fai"
+  - dict:
+      type: file
+      description: Reference sequence dictionary
+      pattern: "*.dict"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test' ]
+  - bed:
+      type: file
+      description: A list of scattered interval lists
+      pattern: "*.interval_list"
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+
+authors:
+  - "@nvnieuwk"

modules/genomescope2/main.nf

@@ -0,0 +1,40 @@
+process GENOMESCOPE2 {
+    tag "$meta.id"
+    label 'process_low'
+
+    conda (params.enable_conda ? "bioconda::genomescope2=2.0" : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/genomescope2:2.0--py310r41hdfd78af_5':
+        'quay.io/biocontainers/genomescope2:2.0--py310r41hdfd78af_5' }"
+
+    input:
+    tuple val(meta), path(histogram)
+
+    output:
+    tuple val(meta), path("*_linear_plot.png")            , emit: linear_plot_png
+    tuple val(meta), path("*_transformed_linear_plot.png"), emit: transformed_linear_plot_png
+    tuple val(meta), path("*_log_plot.png")               , emit: log_plot_png
+    tuple val(meta), path("*_transformed_log_plot.png")   , emit: transformed_log_plot_png
+    tuple val(meta), path("*_model.txt")                  , emit: model
+    tuple val(meta), path("*_summary.txt")                , emit: summary
+    path "versions.yml"                                   , emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    script:
+    def args = task.ext.args ?: ''
+    prefix = task.ext.prefix ?: "${meta.id}"
+    """
+    genomescope2 \\
+        --input $histogram \\
+        $args \\
+        --output . \\
+        --name_prefix $prefix
+
+    cat <<-END_VERSIONS > versions.yml
+    '${task.process}':
+        genomescope2: \$( genomescope2 -v | sed 's/GenomeScope //' )
+    END_VERSIONS
+    """
+}

modules/genomescope2/meta.yml

@@ -0,0 +1,67 @@
+name: "genomescope2"
+description: Estimate genome heterozygosity, repeat content, and size from sequencing reads using a kmer-based statistical approach
+keywords:
+  - "genome size"
+  - "genome heterozygosity"
+  - "repeat content"
+tools:
+  - "genomescope2":
+      description: "Reference-free profiling of polyploid genomes"
+      homepage: "http://qb.cshl.edu/genomescope/genomescope2.0/"
+      documentation: "https://github.com/tbenavi1/genomescope2.0/blob/master/README.md"
+      tool_dev_url: "https://github.com/tbenavi1/genomescope2.0"
+      doi: "https://doi.org/10.1038/s41467-020-14998-3"
+      licence: "['Apache License, Version 2.0 (Apache-2.0)']"
+
+input:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - histogram:
+      type: file
+      description: A K-mer histogram file
+      pattern: "*.hist"
+
+output:
+  - meta:
+      type: map
+      description: |
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - linear_plot_png:
+      type: file
+      description: A genomescope2 linear plot in PNG format
+      pattern: "*_linear_plot.png"
+  - linear_plot_png:
+      type: file
+      description: A genomescope2 linear plot in PNG format
+      pattern: "*_linear_plot.png"
+  - transformed_linear_plot_png:
+      type: file
+      description: A genomescope2 transformed linear plot in PNG format
+      pattern: "*_transformed_linear_plot.png"
+  - log_plot_png:
+      type: file
+      description: A genomescope2 log plot in PNG format
+      pattern: "*_log_plot.png"
+  - transformed_log_plot_png:
+      type: file
+      description: A genomescope2 transformed log plot in PNG format
+      pattern: "*_transformed_log_plot.png"
+  - model:
+      type: file
+      description: Genomescope2 model fit summary
+      pattern: "*_model.txt"
+  - summary:
+      type: file
+      description: Genomescope2 histogram summary
+      pattern: "*_summary.txt"
+
+authors:
+  - "@mahesh-panchal"

modules/rtgtools/vcfeval/main.nf

@@ -35,12 +35,13 @@ process RTGTOOLS_VCFEVAL {
     def eval_regions = evaluation_regions ? "--evaluation-regions=$evaluation_regions" : ""
     def truth_index = truth_vcf_tbi ? "" : "rtg index $truth_vcf"
     def query_index = query_vcf_tbi ? "" : "rtg index $query_vcf"
+    def avail_mem = task.memory.toGiga() + "G"
 
     """
     $truth_index
     $query_index
 
-    rtg vcfeval \\
+    rtg RTG_MEM=$avail_mem vcfeval \\
         $args \\
         --baseline=$truth_vcf \\
         $bed_regions \\

tests/config/pytest_modules.yml

@@ -819,6 +819,10 @@ gatk4/selectvariants:
   - modules/gatk4/selectvariants/**
   - tests/modules/gatk4/selectvariants/**
 
+gatk4/splitintervals:
+  - modules/gatk4/splitintervals/**
+  - tests/modules/gatk4/splitintervals/**
+
 gatk4/splitncigarreads:
   - modules/gatk4/splitncigarreads/**
   - tests/modules/gatk4/splitncigarreads/**
@@ -839,6 +843,10 @@ genmap/mappability:
   - modules/genmap/mappability/**
   - tests/modules/genmap/mappability/**
 
+genomescope2:
+  - modules/genomescope2/**
+  - tests/modules/genomescope2/**
+
 genrich:
   - modules/genrich/**
   - tests/modules/genrich/**
@@ -1647,14 +1655,14 @@ samtools/bam2fq:
   - modules/samtools/bam2fq/**
   - tests/modules/samtools/bam2fq/**
 
-samtools/convert:
-  - modules/samtools/convert/**
-  - tests/modules/samtools/convert/**
-
 samtools/collatefastq:
   - modules/samtools/collatefastq/**
   - tests/modules/samtools/collatefastq/**
 
+samtools/convert:
+  - modules/samtools/convert/**
+  - tests/modules/samtools/convert/**
+
 samtools/depth:
   - modules/samtools/depth/**
   - tests/modules/samtools/depth/**

tests/modules/bowtie2/align/main.nf

@@ -14,9 +14,25 @@ workflow test_bowtie2_align_single_end {
     ]
     fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
     save_unaligned = false
+    sort = false
 
     BOWTIE2_BUILD ( fasta )
-    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned )
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
+}
+
+workflow test_bowtie2_align_single_end_sorted {
+    input = [
+        [ id:'test', single_end:true ], // meta map
+        [
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)
+        ]
+    ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    save_unaligned = false
+    sort = true
+
+    BOWTIE2_BUILD ( fasta )
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
 }
 
 workflow test_bowtie2_align_paired_end {
@@ -29,7 +45,55 @@ workflow test_bowtie2_align_paired_end {
     ]
     fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
     save_unaligned = false
+    sort = false
 
     BOWTIE2_BUILD ( fasta )
-    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned )
-}
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
+}
+
+workflow test_bowtie2_align_paired_end_sorted {
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        [
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+            file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)
+        ]
+    ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    save_unaligned = false
+    sort = true
+
+    BOWTIE2_BUILD ( fasta )
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
+}
+
+workflow test_bowtie2_align_single_end_large_index {
+    input = [
+        [ id:'test', single_end:true ], // meta map
+        [
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)
+        ]
+    ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    save_unaligned = false
+    sort = false
+
+    BOWTIE2_BUILD ( fasta )
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
+}
+
+workflow test_bowtie2_align_paired_end_large_index {
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        [
+            file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true),
+            file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)
+        ]
+    ]
+    fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true)
+    save_unaligned = false
+    sort = false
+
+    BOWTIE2_BUILD ( fasta )
+    BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned, sort )
+}

tests/modules/bowtie2/align/nextflow.config

@@ -5,6 +5,7 @@ params {
 process {
 
     publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
 }
 
 if (params.force_large_index) {

tests/modules/bowtie2/align/test.yml

@@ -1,21 +1,49 @@
 - name: bowtie2 align test_bowtie2_align_single_end
   command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config
   tags:
-    - bowtie2/align
     - bowtie2
+    - bowtie2/align
   files:
-    - path: output/bowtie2/test.bam
-    - path: output/bowtie2/test.bowtie2.log
-      md5sum: 7b8a9e61b7646da1089b041333c41a87
-    - path: output/bowtie2/versions.yml
+    - path: ./output/bowtie2/test.bam
+    - path: ./output/bowtie2/test.bowtie2.log
+    - path: ./output/bowtie2/versions.yml
+
+- name: bowtie2 align test_bowtie2_align_single_end_sorted
+  command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end_sorted -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config
+  tags:
+    - bowtie2
+    - bowtie2/align
+  files:
+    - path: ./output/bowtie2/test.bam
+    - path: ./output/bowtie2/test.bowtie2.log
+    - path: ./output/bowtie2/versions.yml
 
 - name: bowtie2 align test_bowtie2_align_paired_end
-  command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config
+  command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c tests/config/nextflow.config -c tests/modules/bowtie2/align/nextflow.config
   tags:
-    - bowtie2/align
     - bowtie2
+    - bowtie2/align
   files:
-    - path: output/bowtie2/test.bam
-    - path: output/bowtie2/test.bowtie2.log
-      md5sum: bd89ce1b28c93bf822bae391ffcedd19
-    - path: output/bowtie2/versions.yml
+    - path: ./output/bowtie2/test.bam
+    - path: ./output/bowtie2/test.bowtie2.log
+    - path: ./output/bowtie2/versions.yml
+
+- name: bowtie2 align test_bowtie2_align_single_end_large_index
+  command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_single_end_large_index -c tests/config/nextflow.config -c tests/modules/bowtie2/align/nextflow.config --force_large_index
+  tags:
+    - bowtie2
+    - bowtie2/align
+  files:
+    - path: ./output/bowtie2/test.bam
+    - path: ./output/bowtie2/test.bowtie2.log
+    - path: ./output/bowtie2/versions.yml
+
+- name: bowtie2 align test_bowtie2_align_paired_end_large_index
+  command: nextflow run tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end_large_index -c tests/config/nextflow.config -c tests/modules/bowtie2/align/nextflow.config --force_large_index
+  tags:
+    - bowtie2
+    - bowtie2/align
+  files:
+    - path: ./output/bowtie2/test.bam
+    - path: ./output/bowtie2/test.bowtie2.log
+    - path: ./output/bowtie2/versions.yml

tests/modules/gatk4/splitintervals/main.nf

@@ -0,0 +1,33 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { GATK4_SPLITINTERVALS } from '../../../../modules/gatk4/splitintervals/main.nf'
+
+workflow test_gatk4_splitintervals_bed {
+    
+    input = [
+        [ id:'test' ], // meta map
+        file(params.test_data['homo_sapiens']['genome']['genome_multi_interval_bed'], checkIfExists: true)
+    ]
+
+    fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
+    fasta_fai = file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true)
+    fasta_dict = file(params.test_data['homo_sapiens']['genome']['genome_dict'], checkIfExists: true)
+
+    GATK4_SPLITINTERVALS ( input, fasta, fasta_fai, fasta_dict)
+}
+
+workflow test_gatk4_splitintervals_intervals {
+    
+    input = [
+        [ id:'test' ], // meta map
+        file(params.test_data['homo_sapiens']['genome']['genome_interval_list'], checkIfExists: true)
+    ]
+
+    fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true)
+    fasta_fai = file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true)
+    fasta_dict = file(params.test_data['homo_sapiens']['genome']['genome_dict'], checkIfExists: true)
+
+    GATK4_SPLITINTERVALS ( input, fasta, fasta_fai, fasta_dict)
+}

tests/modules/gatk4/splitintervals/nextflow.config

@@ -0,0 +1,9 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+    
+
+    withName: GATK4_SPLITINTERVALS {
+        ext.args = "--scatter-count 2"
+    }
+}

tests/modules/gatk4/splitintervals/test.yml

@@ -0,0 +1,23 @@
+- name: gatk4 splitintervals test_gatk4_splitintervals_bed
+  command: nextflow run tests/modules/gatk4/splitintervals -entry test_gatk4_splitintervals_bed -c tests/config/nextflow.config
+  tags:
+    - gatk4/splitintervals
+    - gatk4
+  files:
+    - path: output/gatk4/test/0000-scattered.interval_list
+      md5sum: c8d6b19e7a92535b6ce9608eae558faa
+    - path: output/gatk4/test/0001-scattered.interval_list
+      md5sum: b1877ad96aec308906594c50ebbe3ded
+    - path: output/gatk4/versions.yml
+
+- name: gatk4 splitintervals test_gatk4_splitintervals_intervals
+  command: nextflow run tests/modules/gatk4/splitintervals -entry test_gatk4_splitintervals_intervals -c tests/config/nextflow.config
+  tags:
+    - gatk4/splitintervals
+    - gatk4
+  files:
+    - path: output/gatk4/test/0000-scattered.interval_list
+      md5sum: ebd6b34a335efc6732ff541936c6d2d5
+    - path: output/gatk4/test/0001-scattered.interval_list
+      md5sum: 9459b0e124fa84ec1e64ac4615bc9af7
+    - path: output/gatk4/versions.yml

tests/modules/genomescope2/main.nf

@@ -0,0 +1,19 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { MERYL_COUNT     } from '../../../modules/meryl/count/main.nf'
+include { MERYL_HISTOGRAM } from '../../../modules/meryl/histogram/main.nf'
+include { GENOMESCOPE2    } from '../../../modules/genomescope2/main.nf'
+
+workflow test_genomescope2 {
+
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true)
+    ]
+
+    MERYL_COUNT ( input )
+    MERYL_HISTOGRAM ( MERYL_COUNT.out.meryl_db )
+    GENOMESCOPE2 ( MERYL_HISTOGRAM.out.hist )
+}

tests/modules/genomescope2/nextflow.config

@@ -0,0 +1,13 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: 'MERYL.*' {
+        ext.args = 'k=21'
+    }
+
+    withName: 'GENOMESCOPE2' {
+        ext.args = '-k 21 -p 1'
+    }
+
+}

tests/modules/genomescope2/test.yml

@@ -0,0 +1,22 @@
+- name: genomescope2 test_genomescope2
+  command: nextflow run tests/modules/genomescope2 -entry test_genomescope2 -c tests/config/nextflow.config
+  tags:
+    - genomescope2
+  files:
+    - path: output/genomescope2/test_linear_plot.png
+      md5sum: 94c165c5028156299a1d4d05766cac51
+    - path: output/genomescope2/test_log_plot.png
+      md5sum: 9d25ca463d92a0c73a893da7fd3979ba
+    - path: output/genomescope2/test_model.txt
+      md5sum: 3caf62f715f64a2f2b8fdff5d079cb84
+    - path: output/genomescope2/test_summary.txt
+      md5sum: 7452860e2cea99b85f3ff60daeac77f5
+    - path: output/genomescope2/test_transformed_linear_plot.png
+      md5sum: 99a64c1c18d8670f64cb863d4334abbb
+    - path: output/genomescope2/test_transformed_log_plot.png
+      md5sum: b4e029c9fb9987ca33b17392a691c1b4
+    - path: output/genomescope2/versions.yml
+      md5sum: 18afeb26f62a47f680b2bb3e27da9cbc
+    - path: output/meryl/test.hist
+      md5sum: f75362ab9cd70d96621b3690e952085f
+    - path: output/meryl/versions.yml