Add a module for sra-tools prefetch (#714)

* chore: apply module template

* refactor: add NCBI settings to options

* docs: complete meta information

* feat: add prefetch process

* fix: correct bash commands

* tests: define the right tests

* style: move option definition to satisfy linting

* fix: extract version correctly

* fix: correct newline issues

* refactor: address review comments

* Apply suggestions from code review

* chore: add retrying via nf-core label

* refactor: validate download thoroughly

* refactor: remove vdb-config input

Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>
This commit is contained in:
Moritz E. Beber 2021-10-11 23:30:41 +02:00 committed by GitHub
parent c19671dca9
commit 07c0830057
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 202 additions and 4 deletions

View file

@ -0,0 +1,78 @@
//
// Utility functions used in nf-core DSL2 module files
//
//
// Extract name of software tool from process name using $task.process
//
def getSoftwareName(task_process) {
return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
}
//
// Extract name of module from process name using $task.process
//
def getProcessName(task_process) {
return task_process.tokenize(':')[-1]
}
//
// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
//
def initOptions(Map args) {
def Map options = [:]
options.args = args.args ?: ''
options.args2 = args.args2 ?: ''
options.args3 = args.args3 ?: ''
options.publish_by_meta = args.publish_by_meta ?: []
options.publish_dir = args.publish_dir ?: ''
options.publish_files = args.publish_files
options.suffix = args.suffix ?: ''
return options
}
//
// Tidy up and join elements of a list to return a path string
//
def getPathFromList(path_list) {
def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries
paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
return paths.join('/')
}
//
// Function to save/publish module results
//
def saveFiles(Map args) {
def ioptions = initOptions(args.options)
def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
// Do not publish versions.yml unless running from pytest workflow
if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
return null
}
if (ioptions.publish_by_meta) {
def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
for (key in key_list) {
if (args.meta && key instanceof String) {
def path = key
if (args.meta.containsKey(key)) {
path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
}
path = path instanceof String ? path : ''
path_list.add(path)
}
}
}
if (ioptions.publish_files instanceof Map) {
for (ext in ioptions.publish_files) {
if (args.filename.endsWith(ext.key)) {
def ext_list = path_list.collect()
ext_list.add(ext.value)
return "${getPathFromList(ext_list)}/$args.filename"
}
}
} else if (ioptions.publish_files == null) {
return "${getPathFromList(path_list)}/$args.filename"
}
}

View file

@ -0,0 +1,50 @@
// Import generic module functions
include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions'
params.options = [:]
options = initOptions(params.options)
process SRATOOLS_PREFETCH {
tag "$id"
label 'process_low'
label 'error_retry'
publishDir "${params.outdir}",
mode: params.publish_dir_mode,
saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
conda (params.enable_conda ? 'bioconda::sra-tools=2.11.0' : null)
if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
container 'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5262h314213e_0'
} else {
container 'quay.io/biocontainers/sra-tools:2.11.0--pl5262h314213e_0'
}
input:
tuple val(meta), val(id)
output:
tuple val(meta), path("$id"), emit: sra
path "versions.yml" , emit: versions
script:
def config = "/LIBS/GUID = \"${UUID.randomUUID().toString()}\"\\n/libs/cloud/report_instance_identity = \"true\"\\n"
"""
eval "\$(vdb-config -o n NCBI_SETTINGS | sed 's/[" ]//g')"
if [[ ! -f "\${NCBI_SETTINGS}" ]]; then
mkdir -p "\$(dirname "\${NCBI_SETTINGS}")"
printf '${config}' > "\${NCBI_SETTINGS}"
fi
prefetch \\
$options.args \\
--progress \\
$id
vdb-validate $id
cat <<-END_VERSIONS > versions.yml
${getProcessName(task.process)}:
${getSoftwareName(task.process)}: \$(prefetch --version 2>&1 | grep -Eo '[0-9.]+')
END_VERSIONS
"""
}

View file

@ -0,0 +1,43 @@
name: sratools_prefetch
description: Download sequencing data from the NCBI Sequence Read Archive (SRA).
keywords:
- sequencing
- fastq
- prefetch
tools:
- sratools:
description: SRA Toolkit and SDK from NCBI
homepage: https://github.com/ncbi/sra-tools
documentation: https://github.com/ncbi/sra-tools/wiki
tool_dev_url: https://github.com/ncbi/sra-tools
licence: ['Public Domain']
input:
- meta:
type: map
description: >
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- id:
type: val
description: >
A string denoting an SRA id.
output:
- meta:
type: map
description: >
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- sra:
type: directory
description: >
Directory containing the ETL data for the given SRA id.
pattern: "*/*.sra"
- versions:
type: file
description: File containing software versions
pattern: "versions.yml"
authors:
- "@Midnighter"

View file

@ -529,6 +529,10 @@ iqtree:
- modules/iqtree/** - modules/iqtree/**
- tests/modules/iqtree/** - tests/modules/iqtree/**
ismapper:
- modules/ismapper/**
- tests/modules/ismapper/**
isoseq3/cluster: isoseq3/cluster:
- modules/isoseq3/cluster/** - modules/isoseq3/cluster/**
- tests/modules/isoseq3/cluster/** - tests/modules/isoseq3/cluster/**
@ -537,10 +541,6 @@ isoseq3/refine:
- modules/isoseq3/refine/** - modules/isoseq3/refine/**
- tests/modules/isoseq3/refine/** - tests/modules/isoseq3/refine/**
ismapper:
- modules/ismapper/**
- tests/modules/ismapper/**
ivar/consensus: ivar/consensus:
- modules/ivar/consensus/** - modules/ivar/consensus/**
- tests/modules/ivar/consensus/** - tests/modules/ivar/consensus/**
@ -979,6 +979,10 @@ spatyper:
- modules/spatyper/** - modules/spatyper/**
- tests/modules/spatyper/** - tests/modules/spatyper/**
sratools/prefetch:
- modules/sratools/prefetch/**
- tests/modules/sratools/prefetch/**
staphopiasccmec: staphopiasccmec:
- modules/staphopiasccmec/** - modules/staphopiasccmec/**
- tests/modules/staphopiasccmec/** - tests/modules/staphopiasccmec/**

View file

@ -0,0 +1,15 @@
#!/usr/bin/env nextflow
nextflow.enable.dsl = 2
include { SRATOOLS_PREFETCH } from '../../../../modules/sratools/prefetch/main.nf' addParams( options: [:] )
workflow test_sratools_prefetch {
input = [
[ id:'test', single_end:false ], // meta map
'ERR2815334'
]
SRATOOLS_PREFETCH ( input )
}

View file

@ -0,0 +1,8 @@
- name: sratools prefetch test_sratools_prefetch
command: nextflow run tests/modules/sratools/prefetch -entry test_sratools_prefetch -c tests/config/nextflow.config
tags:
- sratools/prefetch
- sratools
files:
- path: output/sratools/ERR2815334/ERR2815334.sra
md5sum: 9a98c7f6f4774b7ef94aa915b92a54ea