Add a new module for fasterq-dump (#807)

* chore: use template to create fasterq module

* feat: add fasterq-dump process module

* docs: provide input and output descriptions

* docs: add comment on `--temp`

* fix: use correct variable

* tests: define test output

* refactor: address review comments

* refactor: remove vdb-config input

* chore: add new test data to config

* tests: define single-end and paired-end cases

* refactor: choose specific output

* tests: do not expect single FASTQ for paired-end

* feat: add compression

* Apply suggestions from code review

Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>

* tests: revert the test data name

* Apply suggestions from code review

Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>

modules/sratools/fasterqdump/functions.nf

@@ -0,0 +1,78 @@
+//
+//  Utility functions used in nf-core DSL2 module files
+//
+
+//
+// Extract name of software tool from process name using $task.process
+//
+def getSoftwareName(task_process) {
+    return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
+}
+
+//
+// Extract name of module from process name using $task.process
+//
+def getProcessName(task_process) {
+    return task_process.tokenize(':')[-1]
+}
+
+//
+// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
+//
+def initOptions(Map args) {
+    def Map options = [:]
+    options.args            = args.args ?: ''
+    options.args2           = args.args2 ?: ''
+    options.args3           = args.args3 ?: ''
+    options.publish_by_meta = args.publish_by_meta ?: []
+    options.publish_dir     = args.publish_dir ?: ''
+    options.publish_files   = args.publish_files
+    options.suffix          = args.suffix ?: ''
+    return options
+}
+
+//
+// Tidy up and join elements of a list to return a path string
+//
+def getPathFromList(path_list) {
+    def paths = path_list.findAll { item -> !item?.trim().isEmpty() }      // Remove empty entries
+    paths     = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
+    return paths.join('/')
+}
+
+//
+// Function to save/publish module results
+//
+def saveFiles(Map args) {
+    def ioptions  = initOptions(args.options)
+    def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
+
+    // Do not publish versions.yml unless running from pytest workflow
+    if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
+        return null
+    }
+    if (ioptions.publish_by_meta) {
+        def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
+        for (key in key_list) {
+            if (args.meta && key instanceof String) {
+                def path = key
+                if (args.meta.containsKey(key)) {
+                    path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
+                }
+                path = path instanceof String ? path : ''
+                path_list.add(path)
+            }
+        }
+    }
+    if (ioptions.publish_files instanceof Map) {
+        for (ext in ioptions.publish_files) {
+            if (args.filename.endsWith(ext.key)) {
+                def ext_list = path_list.collect()
+                ext_list.add(ext.value)
+                return "${getPathFromList(ext_list)}/$args.filename"
+            }
+        }
+    } else if (ioptions.publish_files == null) {
+        return "${getPathFromList(path_list)}/$args.filename"
+    }
+}

modules/sratools/fasterqdump/main.nf

@@ -0,0 +1,58 @@
+// Import generic module functions
+include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions'
+
+params.options = [:]
+options        = initOptions(params.options)
+
+process SRATOOLS_FASTERQDUMP {
+    tag "$meta.id"
+    label 'process_medium'
+    publishDir "${params.outdir}",
+        mode: params.publish_dir_mode,
+        saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
+
+    conda (params.enable_conda ? 'bioconda::sra-tools=2.11.0 conda-forge::pigz=2.6' : null)
+    if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
+        container 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0'
+    } else {
+        container 'quay.io/biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0'
+    }
+
+    input:
+    tuple val(meta), path(sra)
+
+    output:
+    tuple val(meta), path(output), emit: reads
+    path "versions.yml"          , emit: versions
+
+    script:
+    def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
+    // Paired-end data extracted by fasterq-dump (--split-3 the default) always creates
+    // *_1.fastq *_2.fastq files but sometimes also an additional *.fastq file
+    // for unpaired reads which we ignore here.
+    output = meta.single_end ? '*.fastq.gz' : '*_{1,2}.fastq.gz'
+    """
+    eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
+    if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
+        mkdir -p "\$(dirname "\${NCBI_SETTINGS}")"
+        printf '${config}' > "\${NCBI_SETTINGS}"
+    fi
+
+    fasterq-dump \\
+        ${options.args} \\
+        --threads $task.cpus \\
+        ${sra.name}
+
+    pigz \\
+        ${options.args2} \\
+        --no-name \\
+        --processes $task.cpus \\
+        *.fastq
+
+    cat <<-END_VERSIONS > versions.yml
+    ${getProcessName(task.process)}:
+        ${getSoftwareName(task.process)}: \$(fasterq-dump --version 2>&1 | grep -Eo '[0-9.]+')
+        pigz: \$( pigz --version 2>&1 | sed 's/pigz //g' )
+    END_VERSIONS
+    """
+}

modules/sratools/fasterqdump/meta.yml

@@ -0,0 +1,42 @@
+name: sratools_fasterqdump
+description: Extract sequencing reads in FASTQ format from a given NCBI Sequence Read Archive (SRA).
+keywords:
+  - sequencing
+  - FASTQ
+  - dump
+tools:
+  - sratools:
+      description: SRA Toolkit and SDK from NCBI
+      homepage: https://github.com/ncbi/sra-tools
+      documentation: https://github.com/ncbi/sra-tools/wiki
+      tool_dev_url: https://github.com/ncbi/sra-tools
+      licence: ['Public Domain']
+
+input:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - sra:
+      type: directory
+      description: Directory containing ETL data for the given SRA.
+      pattern: "*/*.sra"
+
+output:
+  - meta:
+      type: map
+      description: >
+        Groovy Map containing sample information
+        e.g. [ id:'test', single_end:false ]
+  - versions:
+      type: file
+      description: File containing software versions
+      pattern: "versions.yml"
+  - reads:
+      type: file
+      description: Extracted FASTQ file or files if the sequencing reads are paired-end.
+      pattern: "*.fastq.gz"
+
+authors:
+  - "@Midnighter"

tests/config/pytest_modules.yml

@@ -987,6 +987,10 @@ sratools/prefetch:
   - modules/sratools/prefetch/**
   - tests/modules/sratools/prefetch/**
 
+sratools/fasterqdump:
+  - modules/sratools/fasterqdump/**
+  - tests/modules/sratools/fasterqdump/**
+
 staphopiasccmec:
   - modules/staphopiasccmec/**
   - tests/modules/staphopiasccmec/**

tests/config/test_data.config

@@ -80,6 +80,9 @@ params {
                 assembly_gfa                                   = "${test_data_dir}/genomics/sarscov2/illumina/gfa/assembly.gfa"
 
                 test_single_end_bam_readlist_txt               = "${test_data_dir}/genomics/sarscov2/illumina/picard/test.single_end.bam.readlist.txt"
+
+                SRR13255544_tar_gz                             = "${test_data_dir}/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz"
+                SRR11140744_tar_gz                             = "${test_data_dir}/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz"
             }
             'nanopore' {
                 test_sorted_bam                                = "${test_data_dir}/genomics/sarscov2/nanopore/bam/test.sorted.bam"
@@ -155,11 +158,11 @@ params {
                 test2_baserecalibrator_table                   = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test2.baserecalibrator.table"
                 test_pileups_table                             = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test.pileups.table"
                 test2_pileups_table                            = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test2.pileups.table"
-                
+
                 test_test2_paired_mutect2_calls_vcf_gz         = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz"
                 test_test2_paired_mutect2_calls_vcf_gz_tbi     = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.tbi"
                 test_test2_paired_mutect2_calls_vcf_gz_stats   = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.vcf.gz.stats"
-                test_test2_paired_mutect2_calls_f1r2_tar_gz    = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.f1r2.tar.gz"                                                               
+                test_test2_paired_mutect2_calls_f1r2_tar_gz    = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/paired_mutect2_calls/test_test2_paired_mutect2_calls.f1r2.tar.gz"
                 test_test2_paired_mutect2_calls_artifact_prior_tar_gz = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test_test2_paired_mutect2_calls.artifact-prior.tar.gz"
                 test_test2_paired_segmentation_table           = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test_test2_paired.segmentation.table"
                 test_test2_paired_contamination_table          = "${test_data_dir}/genomics/homo_sapiens/illumina/gatk/test_test2_paired.contamination.table"

tests/modules/sratools/fasterqdump/main.nf

@@ -0,0 +1,28 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+include { UNTAR } from '../../../../modules/untar/main.nf'
+include { SRATOOLS_FASTERQDUMP } from '../../../../modules/sratools/fasterqdump/main.nf' addParams( options: [:] )
+
+workflow test_sratools_fasterqdump_single_end {
+
+    archive = file(params.test_data['sarscov2']['illumina']['SRR13255544_tar_gz'], checkIfExists: true)
+    UNTAR ( archive )
+
+    def input = Channel.of([ id:'test_single_end', single_end:true ])
+        .combine(UNTAR.out.untar)
+
+    SRATOOLS_FASTERQDUMP ( input )
+}
+
+workflow test_sratools_fasterqdump_paired_end {
+
+    archive = file(params.test_data['sarscov2']['illumina']['SRR11140744_tar_gz'], checkIfExists: true)
+    UNTAR ( archive )
+
+    def input = Channel.of([ id:'test_paired_end', single_end:false ])
+        .combine(UNTAR.out.untar)
+
+    SRATOOLS_FASTERQDUMP ( input )
+}

tests/modules/sratools/fasterqdump/test.yml

@@ -0,0 +1,23 @@
+- name: sratools fasterqdump test_sratools_fasterqdump_single_end
+  command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_single_end -c tests/config/nextflow.config
+  tags:
+    - sratools
+    - sratools/fasterqdump
+  files:
+    - path: output/sratools/SRR13255544.fastq.gz
+      md5sum: 1054c7b71884acdb5eed8a378f18be82
+    - path: output/untar/SRR13255544/SRR13255544.sra
+      md5sum: 466d05dafb2eec672150754168010b4d
+
+- name: sratools fasterqdump test_sratools_fasterqdump_paired_end
+  command: nextflow run tests/modules/sratools/fasterqdump -entry test_sratools_fasterqdump_paired_end -c tests/config/nextflow.config
+  tags:
+    - sratools
+    - sratools/fasterqdump
+  files:
+    - path: output/sratools/SRR11140744_1.fastq.gz
+      md5sum: 193809c784a4ea132ab2a253fa4f55b6
+    - path: output/sratools/SRR11140744_2.fastq.gz
+      md5sum: 3e3b3af3413f50a1685fd7b3f1456d4e
+    - path: output/untar/SRR11140744/SRR11140744.sra
+      md5sum: 065666caf5b2d5dfb0cb25d5f3abe659