Merge branch 'master' into maxulysse-patch-1

modules/snapaligner/align/main.nf

@@ -1,4 +1,4 @@
-process SNAPALIGNER_PAIRED {
+process SNAPALIGNER_ALIGN {
     tag '$meta.id'
     label 'process_high'
 
@@ -21,15 +21,16 @@ process SNAPALIGNER_PAIRED {
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
+    def subcmd = meta.single_end ? "single" : "paired"
 
     """
     mkdir -p index
     mv $index index/
 
-    snap-aligner paired \\
+    snap-aligner ${subcmd} \\
         index \\
         ${reads.join(" ")} \\
-        -o -bam ${prefix}.bam \\
+        -o ${prefix}.bam \\
         -t ${task.cpus} \\
         $args
 

modules/snapaligner/align/meta.yml

@@ -1,5 +1,5 @@
-name: "snapaligner_paired"
-description: Performs paired end fastq alignment to a fasta reference using SNAP
+name: "snapaligner_align"
+description: Performs fastq alignment to a fasta reference using SNAP
 keywords:
   - alignment
   - map
@@ -22,7 +22,7 @@ input:
         e.g. [ id:'test', single_end:false ]
   - reads:
       type: file
-      description: List of input fastq files of size 2 for fastq or 1 for bam
+      description: List of input fastq files of size 2 for paired fastq or 1 for bam or single fastq
       pattern: "*.{fastq.gz,fq.gz,fastq,fq,bam}"
   - index:
       type: file

modules/snapaligner/single/main.nf

@@ -1,41 +0,0 @@
-process SNAPALIGNER_SINGLE {
-    tag '$meta.id'
-    label 'process_high'
-
-    conda (params.enable_conda ? "bioconda::snap-aligner=2.0.1" : null)
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/snap-aligner:2.0.1--hd03093a_1':
-        'quay.io/biocontainers/snap-aligner:2.0.1--hd03093a_1' }"
-
-    input:
-    tuple val(meta), path(reads)
-    path index
-
-    output:
-    tuple val(meta), path("*.bam"), emit: bam
-    path "versions.yml"           , emit: versions
-
-    when:
-    task.ext.when == null || task.ext.when
-
-    script:
-    def args = task.ext.args ?: ''
-    def prefix = task.ext.prefix ?: "${meta.id}"
-
-    """
-    mkdir -p index
-    mv $index index/
-
-    snap-aligner single \\
-        index \\
-        ${reads.join(" ")} \\
-        -o -bam ${prefix}.bam \\
-        -t ${task.cpus} \\
-        $args
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        snapaligner: \$(snap-aligner 2>&1| head -n 1 | sed 's/^.*version //;s/.\$//')
-    END_VERSIONS
-    """
-}

modules/snapaligner/single/meta.yml

@@ -1,48 +0,0 @@
-name: "snapaligner_single"
-description: Performs single end fastq alignment to a fasta reference using SNAP
-keywords:
-  - alignment
-  - map
-  - fastq
-  - bam
-  - sam
-tools:
-  - "snapaligner":
-      description: "Scalable Nucleotide Alignment Program -- a fast and accurate read aligner for high-throughput sequencing data"
-      homepage: "http://snap.cs.berkeley.edu"
-      documentation: "https://1drv.ms/b/s!AhuEg_0yZD86hcpblUt-muHKYsG8fA?e=R8ogug"
-      tool_dev_url: "https://github.com/amplab/snap"
-      doi: "10.1101/2021.11.23.469039"
-      licence: "['Apache v2']"
-input:
-  - meta:
-      type: map
-      description: |
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - reads:
-      type: file
-      description: List of single end input files
-      pattern: "*.{fastq.gz,fq.gz,fastq,fq,bam}"
-  - index:
-      type: file
-      description: List of SNAP genome index files
-      pattern: "{Genome,GenomeIndex,GenomeIndexHash,OverflowTable}"
-
-output:
-  - meta:
-      type: map
-      description: |
-        Groovy Map containing sample information
-        e.g. [ id:'test', single_end:false ]
-  - versions:
-      type: file
-      description: File containing software versions
-      pattern: "versions.yml"
-  - bam:
-      type: file
-      description: Aligned BAM file
-      pattern: "*.{bam}"
-
-authors:
-  - "@matthdsm"

modules/sratools/prefetch/main.nf

@@ -1,43 +1,26 @@
 process SRATOOLS_PREFETCH {
     tag "$id"
     label 'process_low'
-    label 'error_retry'
 
     conda (params.enable_conda ? 'bioconda::sra-tools=2.11.0' : null)
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5262h314213e_0' :
-        'quay.io/biocontainers/sra-tools:2.11.0--pl5262h314213e_0' }"
+        'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5321ha49a11a_3' :
+        'quay.io/biocontainers/sra-tools:2.11.0--pl5321ha49a11a_3' }"
 
     input:
     tuple val(meta), val(id)
 
     output:
-    tuple val(meta), path("$id"), emit: sra
+    tuple val(meta), path(id), emit: sra
     path "versions.yml"         , emit: versions
 
     when:
     task.ext.when == null || task.ext.when
 
-    script:
-    def args = task.ext.args ?: ''
-    def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
-    """
-    eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
-    if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
-        mkdir -p "\$(dirname "\${NCBI_SETTINGS}")"
-        printf '${config}' > "\${NCBI_SETTINGS}"
-    fi
+    shell:
+    args = task.ext.args ?: ''
+    args2 = task.ext.args2 ?: '5 1 100'  // <num retries> <base delay in seconds> <max delay in seconds>
+    config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
 
-    prefetch \\
-        $args \\
-        --progress \\
-        $id
-
-    vdb-validate $id
-
-    cat <<-END_VERSIONS > versions.yml
-    "${task.process}":
-        sratools: \$(prefetch --version 2>&1 | grep -Eo '[0-9.]+')
-    END_VERSIONS
-    """
+    template 'retry_with_backoff.sh'
 }

modules/sratools/prefetch/templates/retry_with_backoff.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -u
+
+retry_with_backoff() {
+    local max_attempts=${1}
+    local delay=${2}
+    local max_time=${3}
+    local attempt=1
+    local output=
+    local status=
+
+    # Remove the first three arguments to this function in order to access
+    # the 'real' command with `${@}`.
+    shift 3
+
+    while [ ${attempt} -le ${max_attempts} ]; do
+        output=$("${@}")
+        status=${?}
+
+        if [ ${status} -eq 0 ]; then
+            break
+        fi
+
+        if [ ${attempt} -lt ${max_attempts} ]; then
+            echo "Failed attempt ${attempt} of ${max_attempts}. Retrying in ${delay} s." >&2
+            sleep ${delay}
+        elif [ ${attempt} -eq ${max_attempts} ]; then
+            echo "Failed after ${attempt} attempts." >&2
+            return ${status}
+        fi
+
+        attempt=$(( ${attempt} + 1 ))
+        delay=$(( ${delay} * 2 ))
+        if [ ${delay} -ge ${max_time} ]; then
+            delay=${max_time}
+        fi
+    done
+
+    echo "${output}"
+}
+
+eval "$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+if [[ ! -f "${NCBI_SETTINGS}" ]]; then
+    mkdir -p "$(dirname "${NCBI_SETTINGS}")"
+    printf '!{config}' > "${NCBI_SETTINGS}"
+fi
+
+retry_with_backoff !{args2} \
+    prefetch \
+    !{args} \
+    !{id}
+
+vdb-validate !{id}
+
+cat <<-END_VERSIONS > versions.yml
+"!{task.process}":
+    sratools: $(prefetch --version 2>&1 | grep -Eo '[0-9.]+')
+END_VERSIONS

tests/config/pytest_modules.yml

@@ -1767,13 +1767,9 @@ snapaligner/index:
   - modules/snapaligner/index/**
   - tests/modules/snapaligner/index/**
 
-snapaligner/paired:
-  - modules/snapaligner/paired/**
-  - tests/modules/snapaligner/paired/**
-
-snapaligner/single:
-  - modules/snapaligner/single/**
-  - tests/modules/snapaligner/single/**
+snapaligner/align:
+  - modules/snapaligner/align/**
+  - tests/modules/snapaligner/align/**
 
 snpdists:
   - modules/snpdists/**

tests/modules/snapaligner/align/main.nf

@@ -0,0 +1,29 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { SNAPALIGNER_INDEX                       } from '../../../../modules/snapaligner/index/main.nf'
+include { SNAPALIGNER_ALIGN as SNAPALIGNER_SINGLE } from '../../../../modules/snapaligner/align/main.nf'
+include { SNAPALIGNER_ALIGN as SNAPALIGNER_PAIRED } from '../../../../modules/snapaligner/align/main.nf'
+
+workflow test_snapaligner_single {
+
+    input = [
+        [ id:'test', single_end:true ], // meta map
+        [file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)]
+    ]
+
+    SNAPALIGNER_INDEX ( file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),[],[],[])
+    SNAPALIGNER_SINGLE ( input, SNAPALIGNER_INDEX.out.index )
+}
+
+workflow test_snapaligner_paired {
+
+    input = [
+        [ id:'test', single_end:false ], // meta map
+        [file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)]
+    ]
+
+    SNAPALIGNER_INDEX ( file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),[],[],[])
+    SNAPALIGNER_PAIRED ( input, SNAPALIGNER_INDEX.out.index )
+}

tests/modules/snapaligner/align/test.yml

@@ -0,0 +1,19 @@
+- name: snapaligner align test_snapaligner_single
+  command: nextflow run tests/modules/snapaligner/align -entry test_snapaligner_single -c tests/config/nextflow.config
+  tags:
+    - snapaligner/align
+    - snapaligner
+  files:
+    - path: output/snapaligner/test.bam
+      md5sum: 5d95594e4ef1ee23ce56e6a7cb64f0f2
+    - path: output/snapaligner/versions.yml
+
+- name: snapaligner align test_snapaligner_paired
+  command: nextflow run tests/modules/snapaligner/align -entry test_snapaligner_paired -c tests/config/nextflow.config
+  tags:
+    - snapaligner/align
+    - snapaligner
+  files:
+    - path: output/snapaligner/test.bam
+      md5sum: a1405da5876f15dbe8a81516b94c2a15
+    - path: output/snapaligner/versions.yml

tests/modules/snapaligner/paired/main.nf

@@ -1,17 +0,0 @@
-#!/usr/bin/env nextflow
-
-nextflow.enable.dsl = 2
-
-include { SNAPALIGNER_INDEX } from '../../../../modules/snapaligner/index/main.nf'
-include { SNAPALIGNER_PAIRED } from '../../../../modules/snapaligner/paired/main.nf'
-
-workflow test_snapaligner_paired {
-
-    input = [
-        [ id:'test', single_end:false ], // meta map
-        [file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)]
-    ]
-
-    SNAPALIGNER_INDEX ( file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),[],[],[])
-    SNAPALIGNER_PAIRED ( input, SNAPALIGNER_INDEX.out.index )
-}

tests/modules/snapaligner/paired/test.yml

@@ -1,9 +0,0 @@
-- name: snapaligner paired test_snapaligner_paired
-  command: nextflow run tests/modules/snapaligner/paired -entry test_snapaligner_paired -c tests/config/nextflow.config
-  tags:
-    - snapaligner
-    - snapaligner/paired
-  files:
-    - path: output/snapaligner/test.bam
-      md5sum: 2ac92e9539fa246dd6db52b5de56fca5
-    - path: output/snapaligner/versions.yml

tests/modules/snapaligner/single/main.nf

@@ -1,17 +0,0 @@
-#!/usr/bin/env nextflow
-
-nextflow.enable.dsl = 2
-
-include { SNAPALIGNER_INDEX } from '../../../../modules/snapaligner/index/main.nf'
-include { SNAPALIGNER_SINGLE } from '../../../../modules/snapaligner/single/main.nf'
-
-workflow test_snapaligner_single {
-
-    input = [
-        [ id:'test', single_end:false ], // meta map
-        [file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true)]
-    ]
-
-    SNAPALIGNER_INDEX ( file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true),[],[],[])
-    SNAPALIGNER_SINGLE ( input, SNAPALIGNER_INDEX.out.index )
-}

tests/modules/snapaligner/single/nextflow.config

@@ -1,5 +0,0 @@
-process {
-
-    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
-    
-}

tests/modules/snapaligner/single/test.yml

@@ -1,9 +0,0 @@
-- name: snapaligner single test_snapaligner_single
-  command: nextflow run tests/modules/snapaligner/single -entry test_snapaligner_single -c tests/config/nextflow.config
-  tags:
-    - snapaligner/single
-    - snapaligner
-  files:
-    - path: output/snapaligner/test.bam
-      md5sum: 696f7ea8e1aa5f9d7dafb9d0134fe25d
-    - path: output/snapaligner/versions.yml