Merge branch 'master' into antismashlite

modules/bowtie2/align/main.nf

@@ -1,11 +1,11 @@
 process BOWTIE2_ALIGN {
     tag "$meta.id"
-    label 'process_high'
+    label "process_high"
 
-    conda (params.enable_conda ? 'bioconda::bowtie2=2.4.4 bioconda::samtools=1.15.1 conda-forge::pigz=2.6' : null)
+    conda (params.enable_conda ? "bioconda::bowtie2=2.4.4 bioconda::samtools=1.15.1 conda-forge::pigz=2.6" : null)
-    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+    container "${ workflow.containerEngine == "singularity" && !task.ext.singularity_pull_docker_container ?
-        'https://depot.galaxyproject.org/singularity/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0' :
+        "https://depot.galaxyproject.org/singularity/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0" :
-        'quay.io/biocontainers/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0' }"
+        "quay.io/biocontainers/mulled-v2-ac74a7f02cebcfcc07d8e8d1d750af9c83b4d45a:1744f68fe955578c63054b55309e05b41c37a80d-0" }"
 
     input:
     tuple val(meta), path(reads)
@@ -13,69 +13,60 @@ process BOWTIE2_ALIGN {
     val   save_unaligned
 
     output:
-    tuple val(meta), path('*.bam')    , emit: bam
+    tuple val(meta), path("*.bam")    , emit: bam
-    tuple val(meta), path('*.log')    , emit: log
+    tuple val(meta), path("*.log")    , emit: log
-    tuple val(meta), path('*fastq.gz'), emit: fastq, optional:true
+    tuple val(meta), path("*fastq.gz"), emit: fastq, optional:true
     path  "versions.yml"              , emit: versions
 
     when:
     task.ext.when == null || task.ext.when
 
     script:
-    def args = task.ext.args ?: ''
+    def args = task.ext.args ?: ""
-    def args2 = task.ext.args2 ?: ''
+    def args2 = task.ext.args2 ?: ""
     def prefix = task.ext.prefix ?: "${meta.id}"
+
+    def unaligned = ""
+    def reads_args = ""
     if (meta.single_end) {
-        def unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : ''
+        unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : ""
-        """
+        reads_args = "-U ${reads}"
-        INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'`
-        [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'`
-        [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1
-        bowtie2 \\
-            -x \$INDEX \\
-            -U $reads \\
-            --threads $task.cpus \\
-            $unaligned \\
-            $args \\
-            2> ${prefix}.bowtie2.log \\
-            | samtools view -@ $task.cpus $args2 -bhS -o ${prefix}.bam -
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//')
-            samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
-            pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' )
-        END_VERSIONS
-        """
     } else {
-        def unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : ''
+        unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : ""
-        """
+        reads_args = "-1 ${reads[0]} -2 ${reads[1]}"
-        INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'`
-        [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'`
-        [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1
-        bowtie2 \\
-            -x \$INDEX \\
-            -1 ${reads[0]} \\
-            -2 ${reads[1]} \\
-            --threads $task.cpus \\
-            $unaligned \\
-            $args \\
-            2> ${prefix}.bowtie2.log \\
-            | samtools view -@ $task.cpus $args2 -bhS -o ${prefix}.bam -
-
-        if [ -f ${prefix}.unmapped.fastq.1.gz ]; then
-            mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz
-        fi
-        if [ -f ${prefix}.unmapped.fastq.2.gz ]; then
-            mv ${prefix}.unmapped.fastq.2.gz ${prefix}.unmapped_2.fastq.gz
-        fi
-
-        cat <<-END_VERSIONS > versions.yml
-        "${task.process}":
-            bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//')
-            samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
-            pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' )
-        END_VERSIONS
-        """
     }
+
+    def samtools_command = "samtools view -@ $task.cpus --bam --with-header ${args2} > ${prefix}.bam"
+
+
+    """
+    INDEX=`find -L ./ -name "*.rev.1.bt2" | sed "s/.rev.1.bt2//"`
+    [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed "s/.rev.1.bt2l//"`
+    [ -z "\$INDEX" ] && echo "Bowtie2 index files not found" 1>&2 && exit 1
+
+    bowtie2 \\
+        -x \$INDEX \\
+        $reads_args \\
+        --threads $task.cpus \\
+        $unaligned \\
+        $args \\
+        2> ${prefix}.bowtie2.log \\
+        | $samtools_command
+
+    if [ -f ${prefix}.unmapped.fastq.1.gz ]; then
+        mv ${prefix}.unmapped.fastq.1.gz ${prefix}.unmapped_1.fastq.gz
+    fi
+
+    if [ -f ${prefix}.unmapped.fastq.2.gz ]; then
+        mv ${prefix}.unmapped.fastq.2.gz ${prefix}.unmapped_2.fastq.gz
+    fi
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//')
+        samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')
+        pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' )
+    END_VERSIONS
+    """
 }
+

modules/custom/sratoolsncbisettings/main.nf

@@ -0,0 +1,20 @@
+process CUSTOM_SRATOOLSNCBISETTINGS {
+    tag 'ncbi-settings'
+    label 'process_low'
+
+    conda (params.enable_conda ? 'bioconda::sra-tools=2.11.0' : null)
+    container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
+        'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5321ha49a11a_3' :
+        'quay.io/biocontainers/sra-tools:2.11.0--pl5321ha49a11a_3' }"
+
+    output:
+    path('*.mkfg')     , emit: ncbi_settings
+    path 'versions.yml', emit: versions
+
+    when:
+    task.ext.when == null || task.ext.when
+
+    shell:
+    config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
+    template 'detect_ncbi_settings.sh'
+}

modules/custom/sratoolsncbisettings/meta.yml

@@ -0,0 +1,28 @@
+name: "sratoolsncbisettings"
+description: Test for the presence of suitable NCBI settings or create them on the fly.
+keywords:
+  - NCBI
+  - settings
+  - sra-tools
+  - prefetch
+  - fasterq-dump
+tools:
+  - "sratools":
+      description: "SRA Toolkit and SDK from NCBI"
+      homepage: https://github.com/ncbi/sra-tools
+      documentation: https://github.com/ncbi/sra-tools/wiki
+      tool_dev_url: https://github.com/ncbi/sra-tools
+      licence: "['Public Domain']"
+
+output:
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - ncbi_settings:
+      type: file
+      description: An NCBI user settings file.
+      pattern: "*.mkfg"
+
+authors:
+  - "@Midnighter"

modules/custom/sratoolsncbisettings/templates/detect_ncbi_settings.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -u
+
+
+# Get the expected NCBI settings path and define the environment variable
+# `NCBI_SETTINGS`.
+eval "$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+
+# If the user settings do not exist yet, create a file suitable for `prefetch`
+# and `fasterq-dump`. If an existing settings file does not contain the required
+# values, error out with a helpful message.
+if [[ ! -f "${NCBI_SETTINGS}" ]]; then
+    printf '!{config}' > 'user-settings.mkfg'
+else
+    prefetch --help &> /dev/null
+    if [[ $? = 78 ]]; then
+        echo "You have an existing vdb-config at '${NCBI_SETTINGS}' but it is"\
+        "missing the required entries for /LIBS/GUID and"\
+        "/libs/cloud/report_instance_identity."\
+        "Feel free to add the following to your settings file:" >&2
+        echo "$(printf '!{config}')" >&2
+        exit 1
+    fi
+    fasterq-dump --help &> /dev/null
+    if [[ $? = 78 ]]; then
+        echo "You have an existing vdb-config at '${NCBI_SETTINGS}' but it is"\
+        "missing the required entries for /LIBS/GUID and"\
+        "/libs/cloud/report_instance_identity."\
+        "Feel free to add the following to your settings file:" >&2
+        echo "$(printf '!{config}')" >&2
+        exit 1
+    fi
+    if [[ "${NCBI_SETTINGS}" != *.mkfg ]]; then
+        echo "The detected settings '${NCBI_SETTINGS}' do not have the required"\
+        "file extension '.mkfg'." >&2
+        exit 1
+    fi
+    cp "${NCBI_SETTINGS}" ./
+fi
+
+cat <<-END_VERSIONS > versions.yml
+"!{task.process}":
+    sratools: $(vdb-config --version 2>&1 | grep -Eo '[0-9.]+')
+END_VERSIONS

modules/sratools/fasterqdump/main.nf

@@ -9,6 +9,7 @@ process SRATOOLS_FASTERQDUMP {
 
     input:
     tuple val(meta), path(sra)
+    path ncbi_settings
 
     output:
     tuple val(meta), path(output), emit: reads
@@ -20,17 +21,12 @@ process SRATOOLS_FASTERQDUMP {
     script:
     def args = task.ext.args ?: ''
     def args2 = task.ext.args2 ?: ''
-    def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
     // Paired-end data extracted by fasterq-dump (--split-3 the default) always creates
     // *_1.fastq *_2.fastq files but sometimes also an additional *.fastq file
     // for unpaired reads which we ignore here.
     output = meta.single_end ? '*.fastq.gz' : '*_{1,2}.fastq.gz'
     """
-    eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+    export NCBI_SETTINGS="\$PWD/${ncbi_settings}"
-    if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
-        mkdir -p "\$(dirname "\${NCBI_SETTINGS}")"
-        printf '${config}' > "\${NCBI_SETTINGS}"
-    fi
 
     fasterq-dump \\
         $args \\

modules/sratools/fasterqdump/meta.yml

@@ -10,7 +10,7 @@ tools:
       homepage: https://github.com/ncbi/sra-tools
       documentation: https://github.com/ncbi/sra-tools/wiki
       tool_dev_url: https://github.com/ncbi/sra-tools
-      licence: ["US-Government-Work"]
+      licence: ["Public Domain"]
 
 input:
   - meta:
@@ -22,6 +22,11 @@ input:
       type: directory
       description: Directory containing ETL data for the given SRA.
       pattern: "*/*.sra"
+  - ncbi_settings:
+      type: file
+      description: >
+        An NCBI user settings file.
+      pattern: "*.mkfg"
 
 output:
   - meta:

modules/sratools/prefetch/main.nf

@@ -9,10 +9,11 @@ process SRATOOLS_PREFETCH {
 
     input:
     tuple val(meta), val(id)
+    path ncbi_settings
 
     output:
     tuple val(meta), path(id), emit: sra
-    path "versions.yml"         , emit: versions
+    path 'versions.yml'      , emit: versions
 
     when:
     task.ext.when == null || task.ext.when
@@ -20,7 +21,5 @@ process SRATOOLS_PREFETCH {
     shell:
     args = task.ext.args ?: ''
     args2 = task.ext.args2 ?: '5 1 100'  // <num retries> <base delay in seconds> <max delay in seconds>
-    config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
-
     template 'retry_with_backoff.sh'
 }

modules/sratools/prefetch/meta.yml

@@ -10,7 +10,7 @@ tools:
       homepage: https://github.com/ncbi/sra-tools
       documentation: https://github.com/ncbi/sra-tools/wiki
       tool_dev_url: https://github.com/ncbi/sra-tools
-      licence: ["US-Government-Work"]
+      licence: ["Public Domain"]
 
 input:
   - meta:
@@ -22,6 +22,11 @@ input:
       type: val
       description: >
         A string denoting an SRA id.
+  - ncbi_settings:
+      type: file
+      description: >
+        An NCBI user settings file.
+      pattern: "*.mkfg"
 
 output:
   - meta:

modules/sratools/prefetch/templates/retry_with_backoff.sh

@@ -40,11 +40,7 @@ retry_with_backoff() {
     echo "${output}"
 }
 
-eval "$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+export NCBI_SETTINGS="$PWD/!{ncbi_settings}"
-if [[ ! -f "${NCBI_SETTINGS}" ]]; then
-    mkdir -p "$(dirname "${NCBI_SETTINGS}")"
-    printf '!{config}' > "${NCBI_SETTINGS}"
-fi
 
 retry_with_backoff !{args2} \
     prefetch \

subworkflows/nf-core/sra_fastq/main.nf

@@ -1,34 +0,0 @@
-//
-// Download FASTQ sequencing reads from the NCBI's Sequence Read Archive (SRA).
-//
-
-params.prefetch_options    = [:]
-params.fasterqdump_options = [:]
-
-include { SRATOOLS_PREFETCH    } from '../../../modules/sratools/prefetch/main'    addParams( options: params.prefetch_options    )
-include { SRATOOLS_FASTERQDUMP } from '../../../modules/sratools/fasterqdump/main' addParams( options: params.fasterqdump_options )
-
-workflow SRA_FASTQ {
-    take:
-    sra_ids  // channel: [ val(meta), val(id) ]
-
-    main:
-
-    ch_versions = Channel.empty()
-
-    //
-    // Prefetch sequencing reads in SRA format.
-    //
-    SRATOOLS_PREFETCH ( sra_ids )
-    ch_versions = ch_versions.mix( SRATOOLS_PREFETCH.out.versions.first() )
-
-    //
-    // Convert the SRA format into one or more compressed FASTQ files.
-    //
-    SRATOOLS_FASTERQDUMP ( SRATOOLS_PREFETCH.out.sra )
-    ch_versions = ch_versions.mix( SRATOOLS_FASTERQDUMP.out.versions.first() )
-
-    emit:
-    reads    = SRATOOLS_FASTERQDUMP.out.reads  // channel: [ val(meta), [ reads ] ]
-    versions = ch_versions                     // channel: [ versions.yml ]
-}

subworkflows/nf-core/srafastq/main.nf

@@ -0,0 +1,38 @@
+include { CUSTOM_SRATOOLSNCBISETTINGS } from '../../../modules/custom/sratoolsncbisettings/main'
+include { SRATOOLS_PREFETCH           } from '../../../modules/sratools/prefetch/main'
+include { SRATOOLS_FASTERQDUMP        } from '../../../modules/sratools/fasterqdump/main'
+
+/**
+ * Download FASTQ sequencing reads from the NCBI's Sequence Read Archive (SRA).
+ */
+workflow SRAFASTQ {
+    take:
+    sra_ids  // channel: [ val(meta), val(id) ]
+
+    main:
+
+    ch_versions = Channel.empty()
+
+    //
+    // Detect existing NCBI user settings or create new ones.
+    //
+    CUSTOM_SRATOOLSNCBISETTINGS()
+    def settings = CUSTOM_SRATOOLSNCBISETTINGS.out.ncbi_settings
+    ch_versions = ch_versions.mix( CUSTOM_SRATOOLSNCBISETTINGS.out.versions )
+
+    //
+    // Prefetch sequencing reads in SRA format.
+    //
+    SRATOOLS_PREFETCH ( sra_ids, settings )
+    ch_versions = ch_versions.mix( SRATOOLS_PREFETCH.out.versions.first() )
+
+    //
+    // Convert the SRA format into one or more compressed FASTQ files.
+    //
+    SRATOOLS_FASTERQDUMP ( SRATOOLS_PREFETCH.out.sra, settings )
+    ch_versions = ch_versions.mix( SRATOOLS_FASTERQDUMP.out.versions.first() )
+
+    emit:
+    reads    = SRATOOLS_FASTERQDUMP.out.reads  // channel: [ val(meta), [ reads ] ]
+    versions = ch_versions                     // channel: [ versions.yml ]
+}

subworkflows/nf-core/srafastq/meta.yml

@@ -1,11 +1,14 @@
 name: sra_fastq
 description: Download FASTQ sequencing reads from the NCBI's Sequence Read Archive (SRA).
 keywords:
+  - SRA
+  - NCBI
   - sequencing
   - FASTQ
   - prefetch
-  - dump
+  - fasterq-dump
 modules:
+  - custom/sratoolsncbisettings
   - sratools/prefetch
   - sratools/fasterqdump
 input:
@@ -17,7 +20,7 @@ input:
   - id:
       type: string
       description: >
-        SRA identifier.
+        SRA run identifier.
 # TODO Update when we decide on a standard for subworkflow docs
 output:
   - meta:

tests/config/pytest_modules.yml

@@ -499,6 +499,10 @@ custom/getchromsizes:
   - modules/custom/getchromsizes/**
   - tests/modules/custom/getchromsizes/**
 
+custom/sratoolsncbisettings:
+  - modules/custom/sratoolsncbisettings/**
+  - tests/modules/custom/sratoolsncbisettings/**
+
 cutadapt:
   - modules/cutadapt/**
   - tests/modules/cutadapt/**

tests/config/test_data.config

@@ -382,7 +382,7 @@ params {
                 test3_gff                       = "${test_data_dir}/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/genome/gff/test3.gff"
             }
             'illumina' {
-                test_1_fastq_gz                 = "${test_data_dir}/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fasta/test_1.fastq.gz"
+                test_1_fastq_gz                 = "${test_data_dir}/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_1.fastq.gz"
                 test_2_fastq_gz                 = "${test_data_dir}/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_2.fastq.gz"
                 test_se_fastq_gz                = "${test_data_dir}/genomics/prokaryotes/candidatus_portiera_aleyrodidarum/illumina/fastq/test_se.fastq.gz"
             }
@@ -426,6 +426,9 @@ params {
                 test_merge_cool_cp2             = "${test_data_dir}/genomics/homo_sapiens/cooler/merge/toy/toy.symm.upper.2.cp2.cool"
 
             }
+            'config' {
+                ncbi_user_settings              = "${test_data_dir}/generic/config/ncbi_user_settings.mkfg"
+            }
         }
     }
 }

tests/modules/bowtie2/align/test.yml

@@ -1,83 +1,21 @@
-- name: bowtie2 align single-end
+- name: bowtie2 align test_bowtie2_align_single_end
   command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config
   tags:
-    - bowtie2
     - bowtie2/align
+    - bowtie2
   files:
-    - path: ./output/bowtie2/test.bam
+    - path: output/bowtie2/test.bam
-    - path: ./output/bowtie2/test.bowtie2.log
+    - path: output/bowtie2/test.bowtie2.log
-    - path: ./output/bowtie2/bowtie2/genome.3.bt2
+      md5sum: 7b8a9e61b7646da1089b041333c41a87
-      md5sum: 4ed93abba181d8dfab2e303e33114777
+    - path: output/bowtie2/versions.yml
-    - path: ./output/bowtie2/bowtie2/genome.2.bt2
-      md5sum: 47b153cd1319abc88dda532462651fcf
-    - path: ./output/bowtie2/bowtie2/genome.1.bt2
-      md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf
-    - path: ./output/bowtie2/bowtie2/genome.4.bt2
-      md5sum: c25be5f8b0378abf7a58c8a880b87626
-    - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2
-      md5sum: 52be6950579598a990570fbcf5372184
-    - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2
-      md5sum: e3b4ef343dea4dd571642010a7d09597
 
-- name: bowtie2 align paired-end
+- name: bowtie2 align test_bowtie2_align_paired_end
   command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config
   tags:
-    - bowtie2
     - bowtie2/align
-  files:
-    - path: ./output/bowtie2/test.bam
-    - path: ./output/bowtie2/test.bowtie2.log
-    - path: ./output/bowtie2/bowtie2/genome.3.bt2
-      md5sum: 4ed93abba181d8dfab2e303e33114777
-    - path: ./output/bowtie2/bowtie2/genome.2.bt2
-      md5sum: 47b153cd1319abc88dda532462651fcf
-    - path: ./output/bowtie2/bowtie2/genome.1.bt2
-      md5sum: cbe3d0bbea55bc57c99b4bfa25b5fbdf
-    - path: ./output/bowtie2/bowtie2/genome.4.bt2
-      md5sum: c25be5f8b0378abf7a58c8a880b87626
-    - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2
-      md5sum: 52be6950579598a990570fbcf5372184
-    - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2
-      md5sum: e3b4ef343dea4dd571642010a7d09597
-
-- name: bowtie2 align single-end large-index
-  command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index
-  tags:
     - bowtie2
-    - bowtie2/align
   files:
-    - path: ./output/bowtie2/test.bam
+    - path: output/bowtie2/test.bam
-    - path: ./output/bowtie2/test.bowtie2.log
+    - path: output/bowtie2/test.bowtie2.log
-    - path: ./output/bowtie2/bowtie2/genome.3.bt2l
+      md5sum: bd89ce1b28c93bf822bae391ffcedd19
-      md5sum: 8952b3e0b1ce9a7a5916f2e147180853
+    - path: output/bowtie2/versions.yml
-    - path: ./output/bowtie2/bowtie2/genome.2.bt2l
-      md5sum: 22c284084784a0720989595e0c9461fd
-    - path: ./output/bowtie2/bowtie2/genome.1.bt2l
-      md5sum: 07d811cd4e350d56267183d2ac7023a5
-    - path: ./output/bowtie2/bowtie2/genome.4.bt2l
-      md5sum: c25be5f8b0378abf7a58c8a880b87626
-    - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l
-      md5sum: fda48e35925fb24d1c0785f021981e25
-    - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l
-      md5sum: 802c26d32b970e1b105032b7ce7348b4
-
-- name: bowtie2 align paired-end large-index
-  command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index
-  tags:
-    - bowtie2
-    - bowtie2/align
-  files:
-    - path: ./output/bowtie2/test.bam
-    - path: ./output/bowtie2/test.bowtie2.log
-    - path: ./output/bowtie2/bowtie2/genome.3.bt2l
-      md5sum: 8952b3e0b1ce9a7a5916f2e147180853
-    - path: ./output/bowtie2/bowtie2/genome.2.bt2l
-      md5sum: 22c284084784a0720989595e0c9461fd
-    - path: ./output/bowtie2/bowtie2/genome.1.bt2l
-      md5sum: 07d811cd4e350d56267183d2ac7023a5
-    - path: ./output/bowtie2/bowtie2/genome.4.bt2l
-      md5sum: c25be5f8b0378abf7a58c8a880b87626
-    - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l
-      md5sum: fda48e35925fb24d1c0785f021981e25
-    - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l
-      md5sum: 802c26d32b970e1b105032b7ce7348b4

tests/modules/custom/sratoolsncbisettings/main.nf

@@ -0,0 +1,44 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { CUSTOM_SRATOOLSNCBISETTINGS } from '../../../../modules/custom/sratoolsncbisettings/main.nf'
+
+workflow test_sratoolsncbisettings_with_good_existing {
+
+    file(params.settings_path).mkdirs()
+    def settings = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true)
+    settings.copyTo(params.settings_file)
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}
+
+workflow test_sratoolsncbisettings_with_bad_existing {
+
+    file(params.settings_path).mkdirs()
+    def settings = file(params.settings_file)
+    settings.text = '''
+    ## auto-generated configuration file - DO NOT EDIT ##
+
+    config/default = "false"
+    /repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
+    /repository/remote/protected/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi"
+    /repository/user/ad/public/apps/file/volumes/flatAd = "."
+    /repository/user/ad/public/apps/refseq/volumes/refseqAd = "."
+    /repository/user/ad/public/apps/sra/volumes/sraAd = "."
+    /repository/user/ad/public/apps/sraPileup/volumes/ad = "."
+    /repository/user/ad/public/apps/sraRealign/volumes/ad = "."
+    /repository/user/ad/public/apps/wgs/volumes/wgsAd = "."
+    /repository/user/ad/public/root = "."
+    /repository/user/default-path = "/root/ncbi"
+    '''.stripIndent()
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}
+
+workflow test_sratoolsncbisettings_with_nonexisting {
+    def settings = file(params.settings_file)
+    settings.delete()
+
+    CUSTOM_SRATOOLSNCBISETTINGS()
+}

tests/modules/custom/sratoolsncbisettings/nextflow.config

@@ -0,0 +1,8 @@
+params.settings_path = '/tmp/.ncbi'
+params.settings_file = "${params.settings_path}/user-settings.mkfg"
+
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+}

tests/modules/custom/sratoolsncbisettings/nextflow_mount.config

@@ -0,0 +1,17 @@
+params.settings_path = '/tmp/.ncbi'
+params.settings_file = "${params.settings_path}/user-settings.mkfg"
+
+env.NCBI_SETTINGS = params.settings_file
+
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+    withName: CUSTOM_SRATOOLSNCBISETTINGS {
+        containerOptions = {
+            (workflow.containerEngine == 'singularity') ?
+            "-B ${params.settings_path}:${params.settings_path}" :
+            "-v ${params.settings_path}:${params.settings_path}"
+        }
+    }
+}

tests/modules/custom/sratoolsncbisettings/test.yml

@@ -0,0 +1,44 @@
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_good_existing"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_good_existing -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow_mount.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      md5sum: 955e27aff2c277c2f1f0943a098888c1
+    - path: output/custom/versions.yml
+      contains:
+        - "sratools: 2.11.0"
+
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_bad_existing"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_bad_existing -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow_mount.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  exit_code: 1
+  stdout:
+    contains:
+      - "Command error:"
+      - "missing the required entries"
+      - "/LIBS/GUID"
+      - "/libs/cloud/report_instance_identity"
+      - "Feel free to add the following"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      should_exist: false
+    - path: output/custom/versions.yml
+      should_exist: false
+
+- name: "custom sratoolsncbisettings test_sratoolsncbisettings_with_nonexisting"
+  command: nextflow run ./tests/modules/custom/sratoolsncbisettings -entry test_sratoolsncbisettings_with_nonexisting -c ./tests/config/nextflow.config -c ./tests/modules/custom/sratoolsncbisettings/nextflow.config
+  tags:
+    - "custom"
+    - "custom/sratoolsncbisettings"
+  files:
+    - path: "output/custom/user-settings.mkfg"
+      contains:
+        - "/LIBS/GUID"
+        - "/libs/cloud/report_instance_identity"
+    - path: output/custom/versions.yml
+      contains:
+        - "sratools: 2.11.0"

tests/modules/sratools/fasterqdump/main.nf

@@ -13,7 +13,7 @@ workflow test_sratools_fasterqdump_single_end {
     def input = Channel.of([ id:'test_single_end', single_end:true ])
         .combine(UNTAR.out.untar.map{ it[1] })
 
-    SRATOOLS_FASTERQDUMP ( input )
+    SRATOOLS_FASTERQDUMP(input, file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true))
 }
 
 workflow test_sratools_fasterqdump_paired_end {
@@ -24,5 +24,5 @@ workflow test_sratools_fasterqdump_paired_end {
     def input = Channel.of([ id:'test_paired_end', single_end:false ])
         .combine(UNTAR.out.untar.map{ it[1] })
 
-    SRATOOLS_FASTERQDUMP ( input )
+    SRATOOLS_FASTERQDUMP(input, file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true))
 }

tests/modules/sratools/fasterqdump/test.yml

@@ -8,6 +8,9 @@
       md5sum: 1054c7b71884acdb5eed8a378f18be82
     - path: output/untar/SRR13255544/SRR13255544.sra
       md5sum: 466d05dafb2eec672150754168010b4d
+    - path: output/sratools/versions.yml
+      contains:
+        - "sratools: 2.11.0"
 
 - name: sratools fasterqdump test_sratools_fasterqdump_paired_end
   command: nextflow run ./tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/sratools/fasterqdump/nextflow.config
@@ -21,3 +24,6 @@
       md5sum: 3e3b3af3413f50a1685fd7b3f1456d4e
     - path: output/untar/SRR11140744/SRR11140744.sra
       md5sum: 065666caf5b2d5dfb0cb25d5f3abe659
+    - path: output/sratools/versions.yml
+      contains:
+        - "sratools: 2.11.0"

tests/modules/sratools/prefetch/main.nf

@@ -8,8 +8,8 @@ workflow test_sratools_prefetch {
 
     input = [
         [ id:'test', single_end:false ], // meta map
-        'ERR2815334'
+        'DRR000774'
     ]
 
-    SRATOOLS_PREFETCH ( input )
+    SRATOOLS_PREFETCH(input, file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true))
 }

tests/modules/sratools/prefetch/test.yml

@@ -4,5 +4,8 @@
     - sratools/prefetch
     - sratools
   files:
-    - path: output/sratools/ERR2815334/ERR2815334.sra
+    - path: output/sratools/DRR000774/DRR000774.sra
-      md5sum: 9a98c7f6f4774b7ef94aa915b92a54ea
+      md5sum: 7647dba20c89c0e3d7ad13842f060eb0
+    - path: output/sratools/versions.yml
+      contains:
+        - "sratools: 2.11.0"

tests/subworkflows/nf-core/sra_fastq/main.nf

@@ -1,23 +0,0 @@
-#!/usr/bin/env nextflow
-
-nextflow.enable.dsl = 2
-
-include { SRA_FASTQ } from '../../../../subworkflows/nf-core/sra_fastq/main.nf' addParams( [:] )
-
-workflow test_sra_fastq_single_end {
-    input = [
-        [ id:'test_single_end', single_end:true ], // meta map
-        'SRR13255544'
-    ]
-
-    SRA_FASTQ ( input )
-}
-
-workflow test_sra_fastq_paired_end {
-    input = [
-        [ id:'test_paired_end', single_end:false ], // meta map
-        'SRR11140744'
-    ]
-
-    SRA_FASTQ ( input )
-}

tests/subworkflows/nf-core/sra_fastq/test.yml

@@ -1,27 +0,0 @@
-- name: sra fastq single-end
-  command: nextflow run ./tests/subworkflows/nf-core/sra_fastq -entry test_sra_fastq_single_end -c tests/config/nextflow.config
-  tags:
-    - subworkflows
-    # - subworkflows/sra_fastq
-    # Modules
-    # - sratools
-    # - sratools/prefetch
-    # - sratools/fasterqdump
-  files:
-    - path: output/sratools/SRR13255544.fastq.gz
-      md5sum: 1054c7b71884acdb5eed8a378f18be82
-
-- name: sra fastq paired-end
-  command: nextflow run ./tests/subworkflows/nf-core/sra_fastq -entry test_sra_fastq_paired_end -c tests/config/nextflow.config
-  tags:
-    - subworkflows
-    # - subworkflows/sra_fastq
-    # Modules
-    # - sratools
-    # - sratools/prefetch
-    # - sratools/fasterqdump
-  files:
-    - path: output/sratools/SRR11140744_1.fastq.gz
-      md5sum: 193809c784a4ea132ab2a253fa4f55b6
-    - path: output/sratools/SRR11140744_2.fastq.gz
-      md5sum: 3e3b3af3413f50a1685fd7b3f1456d4e

tests/subworkflows/nf-core/srafastq/main.nf

@@ -0,0 +1,29 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { SRAFASTQ } from '../../../../subworkflows/nf-core/srafastq/main.nf'
+
+workflow test_srafastq_single_end {
+    input = Channel.of(
+        [
+            [ id:'test_single_end1', single_end:true ], // meta map
+            'DRR000774'
+        ],
+        [
+            [ id:'test_single_end2', single_end:true ], // meta map
+            'DRR000775'
+        ]
+    )
+
+    SRAFASTQ ( input )
+}
+
+workflow test_srafastq_paired_end {
+    input = [
+        [ id:'test_paired_end', single_end:false ], // meta map
+        'SRR11140744'
+    ]
+
+    SRAFASTQ ( input )
+}

tests/subworkflows/nf-core/srafastq/nextflow.config

@@ -0,0 +1,5 @@
+process {
+
+    publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }
+
+}

tests/subworkflows/nf-core/srafastq/test.yml

@@ -0,0 +1,29 @@
+- name: srafastq single-end
+  command: nextflow run ./tests/subworkflows/nf-core/srafastq -entry test_srafastq_single_end -c tests/config/nextflow.config -c tests/subworkflows/nf-core/srafastq/nextflow.config
+  tags:
+    - subworkflows
+    # - subworkflows/srafastq
+    # Modules
+    # - sratools
+    # - sratools/prefetch
+    # - sratools/fasterqdump
+  files:
+    - path: output/sratools/DRR000774.fastq.gz
+      md5sum: 19029a1132115b55277a0d79ee089b49
+    - path: output/sratools/DRR000775.fastq.gz
+      md5sum: 59ff24c86ecb260752668c059c2a1eaf
+
+- name: srafastq paired-end
+  command: nextflow run ./tests/subworkflows/nf-core/srafastq -entry test_srafastq_paired_end -c tests/config/nextflow.config -c tests/subworkflows/nf-core/srafastq/nextflow.config
+  tags:
+    - subworkflows
+    # - subworkflows/srafastq
+    # Modules
+    # - sratools
+    # - sratools/prefetch
+    # - sratools/fasterqdump
+  files:
+    - path: output/sratools/SRR11140744_1.fastq.gz
+      md5sum: 193809c784a4ea132ab2a253fa4f55b6
+    - path: output/sratools/SRR11140744_2.fastq.gz
+      md5sum: 3e3b3af3413f50a1685fd7b3f1456d4e