Add new module: Das Tool (#1004)

* add pydamage module

* remove TODOs

* split module by subcommands

* update version parsing

* remove forgotten TODOs

* update module names

* remove old holistic module

* Update modules/pydamage/analyze/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* add keywords

* update resource requirement

* Update modules/pydamage/filter/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/pydamage/filter/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* merge from upstream

* update pydamage from upstream

* add freebayes

* update pydamage test from upstream

* fix meta.yml

* update functions.nf

* update test.yml

* update version parsing

* update version parsing

* fix indentation

* Update modules/freebayes/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/freebayes/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/freebayes/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* add optional inputs

* Update modules/freebayes/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* add bed test

* add metabat2 module

* only freebayes

* remove metabat2

* update md5sum because of vcf including date of the day

* add keyword

* rescue conflicted files

* attempt to fix ECLint

* add pytest workflow for metabat

* remove -

* Update modules/metabat2/jgisummarizebamcontigdepths/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/metabat2/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/metabat2/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/jgisummarizebamcontigdepths/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* add optional inputs/outpus

* remove trailing whitespace

* compressing and removing not reproducible md5sums

* follow symlinks while decompressing

* add dastool/scaffolds2bin

* add dastool

* remove non reproducible md5sum check for compressed files

* Update tests/modules/metabat2/metabat2/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update tests/modules/metabat2/metabat2/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* split tests

* export env variable

* remove metabat2 from PR

* fix linting errors

* remove traling whitespace

* Update modules/metabat2/jgisummarizebamcontigdepths/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/jgisummarizebamcontigdepths/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/metabat2/main.nf

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* Update modules/metabat2/metabat2/meta.yml

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>

* answer PR comments and switch to bgzip

* PR review updates

* update from PR review

* update test files

* add bacillus fragilis alignments

* switch tests to bacillus fragilis

* add string check

* update test string

* add pr comment answer

* last fixes for PR review

Co-authored-by: James A. Fellows Yates <jfy133@gmail.com>
Co-authored-by: Harshil Patel <drpatelh@users.noreply.github.com>
Co-authored-by: Gregor Sturm <mail@gregor-sturm.de>
This commit is contained in:
Maxime Borry 2021-11-19 08:33:29 +01:00 committed by GitHub
parent 19035c99d1
commit 72c94dbed9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 542 additions and 0 deletions

View file

@ -0,0 +1,78 @@
//
// Utility functions used in nf-core DSL2 module files
//
//
// Extract name of software tool from process name using $task.process
//
def getSoftwareName(task_process) {
return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
}
//
// Extract name of module from process name using $task.process
//
def getProcessName(task_process) {
return task_process.tokenize(':')[-1]
}
//
// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
//
def initOptions(Map args) {
def Map options = [:]
options.args = args.args ?: ''
options.args2 = args.args2 ?: ''
options.args3 = args.args3 ?: ''
options.publish_by_meta = args.publish_by_meta ?: []
options.publish_dir = args.publish_dir ?: ''
options.publish_files = args.publish_files
options.suffix = args.suffix ?: ''
return options
}
//
// Tidy up and join elements of a list to return a path string
//
def getPathFromList(path_list) {
def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries
paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
return paths.join('/')
}
//
// Function to save/publish module results
//
def saveFiles(Map args) {
def ioptions = initOptions(args.options)
def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
// Do not publish versions.yml unless running from pytest workflow
if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
return null
}
if (ioptions.publish_by_meta) {
def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
for (key in key_list) {
if (args.meta && key instanceof String) {
def path = key
if (args.meta.containsKey(key)) {
path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
}
path = path instanceof String ? path : ''
path_list.add(path)
}
}
}
if (ioptions.publish_files instanceof Map) {
for (ext in ioptions.publish_files) {
if (args.filename.endsWith(ext.key)) {
def ext_list = path_list.collect()
ext_list.add(ext.value)
return "${getPathFromList(ext_list)}/$args.filename"
}
}
} else if (ioptions.publish_files == null) {
return "${getPathFromList(path_list)}/$args.filename"
}
}

View file

@ -0,0 +1,73 @@
include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions'
params.options = [:]
options = initOptions(params.options)
process DASTOOL_DASTOOL {
tag "$meta.id"
label 'process_medium'
publishDir "${params.outdir}",
mode: params.publish_dir_mode,
saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
conda (params.enable_conda ? "bioconda::das_tool=1.1.3" : null)
if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
container "https://depot.galaxyproject.org/singularity/das_tool:1.1.3--r41hdfd78af_0"
} else {
container "quay.io/biocontainers/das_tool:1.1.3--r41hdfd78af_0"
}
input:
tuple val(meta), path(contigs), path(bins)
path(proteins)
path(db_directory)
val(search_engine)
output:
tuple val(meta), path("*.log") , emit: log
tuple val(meta), path("*_summary.txt") , emit: summary
tuple val(meta), path("*_DASTool_scaffolds2bin.txt") , emit: scaffolds2bin
tuple val(meta), path("*.eval") , optional: true, emit: eval
tuple val(meta), path("*_DASTool_bins/*.fa") , optional: true, emit: bins
tuple val(meta), path("*.pdf") , optional: true, emit: pdfs
tuple val(meta), path("*.proteins.faa") , optional: true, emit: fasta_proteins
tuple val(meta), path("*.archaea.scg") , optional: true, emit: fasta_archaea_scg
tuple val(meta), path("*.bacteria.scg") , optional: true, emit: fasta_bacteria_scg
tuple val(meta), path("*.seqlength") , optional: true, emit: seqlength
path "versions.yml" , emit: versions
script:
def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
def bin_list = bins instanceof List ? bins.join(",") : "$bins"
def engine = search_engine ? "--search_engine $search_engine" : "--search_engine diamond"
def db_dir = db_directory ? "--db_directory $db_directory" : ""
def clean_contigs = contigs.toString() - ".gz"
def decompress_contigs = contigs.toString() == clean_contigs ? "" : "gunzip -q -f $contigs"
def decompress_proteins = proteins ? "gunzip -f $proteins" : ""
def clean_proteins = proteins ? proteins.toString() - ".gz" : ""
def proteins_pred = proteins ? "--proteins $clean_proteins" : ""
if (! search_engine) {
log.info('[DAS_Tool] Default search engine (USEARCH) is proprietary software and not available in bioconda. Using DIAMOND as alternative.')
}
"""
$decompress_proteins
$decompress_contigs
DAS_Tool \\
$options.args \\
$proteins_pred \\
$db_dir \\
$engine \\
-t $task.cpus \\
--bins $bin_list \\
-c $clean_contigs \\
-o $prefix
cat <<-END_VERSIONS > versions.yml
${getProcessName(task.process)}:
${getSoftwareName(task.process)}: \$( DAS_Tool --version 2>&1 | grep "DAS Tool" | sed 's/DAS Tool version //' )
END_VERSIONS
"""
}

View file

@ -0,0 +1,100 @@
name: dastool_dastool
description: DAS Tool binning step.
keywords:
- binning
- das tool
- table
- de novo
- bins
- contigs
- assembly
- das_tool
tools:
- dastool:
description: |
DAS Tool is an automated method that integrates the results
of a flexible number of binning algorithms to calculate an optimized, non-redundant
set of bins from a single assembly.
homepage: https://github.com/cmks/DAS_Tool
documentation: https://github.com/cmks/DAS_Tool
tool_dev_url: https://github.com/cmks/DAS_Tool
doi: "10.1038/s41564-018-0171-1"
licence: ['BSD']
input:
- meta:
type: map
description: |
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- contigs:
type: file
description: fasta file
pattern: "*.{fa.gz,fas.gz,fasta.gz}"
- bins:
type: file
description: "Scaffolds2bin tabular file generated with dastool/scaffolds2bin"
pattern: "*.scaffolds2bin.tsv"
- proteins:
type: file
description: Predicted proteins in prodigal fasta format (>scaffoldID_geneNo)
pattern: "*.{fa.gz,fas.gz,fasta.gz}"
- db_directory:
type: file
description: (optional) Directory of single copy gene database.
- search_engine:
type: val
description: Engine used for single copy gene identification. USEARCH is not supported due to it being proprietary [blast/diamond]
output:
- meta:
type: map
description: |
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- version:
type: file
description: File containing software version
pattern: "versions.yml"
- log:
type: file
description: Log file of the run
pattern: "*.log"
- summary:
type: file
description: Summary of output bins including quality and completeness estimates
pattern: "*summary.txt"
- scaffolds2bin:
type: file
description: Scaffolds to bin file of output bins
pattern: "*.scaffolds2bin.txt"
- eval:
type: file
description: Quality and completeness estimates of input bin sets
pattern: "*.eval"
- pdfs:
type: file
description: Plots showing the amount of high quality bins and score distribution of bins per method
pattern: "*.pdf"
- fasta_proteins:
type: file
description: Output from prodigal if not already supplied
pattern: "*.proteins.faa"
- fasta_archaea_scg:
type: file
description: Results of archaeal single-copy-gene prediction
pattern: "*.archaea.scg"
- fasta_bacteria_scg:
type: file
description: Results of bacterial single-copy-gene prediction
pattern: "*.bacteria.scg"
- seqlength:
type: file
description: Summary of contig lengths
pattern: "*.seqlength"
authors:
- "@maxibor"
- "@jfy133"

View file

@ -0,0 +1,78 @@
//
// Utility functions used in nf-core DSL2 module files
//
//
// Extract name of software tool from process name using $task.process
//
def getSoftwareName(task_process) {
return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
}
//
// Extract name of module from process name using $task.process
//
def getProcessName(task_process) {
return task_process.tokenize(':')[-1]
}
//
// Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
//
def initOptions(Map args) {
def Map options = [:]
options.args = args.args ?: ''
options.args2 = args.args2 ?: ''
options.args3 = args.args3 ?: ''
options.publish_by_meta = args.publish_by_meta ?: []
options.publish_dir = args.publish_dir ?: ''
options.publish_files = args.publish_files
options.suffix = args.suffix ?: ''
return options
}
//
// Tidy up and join elements of a list to return a path string
//
def getPathFromList(path_list) {
def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries
paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes
return paths.join('/')
}
//
// Function to save/publish module results
//
def saveFiles(Map args) {
def ioptions = initOptions(args.options)
def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
// Do not publish versions.yml unless running from pytest workflow
if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
return null
}
if (ioptions.publish_by_meta) {
def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
for (key in key_list) {
if (args.meta && key instanceof String) {
def path = key
if (args.meta.containsKey(key)) {
path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
}
path = path instanceof String ? path : ''
path_list.add(path)
}
}
}
if (ioptions.publish_files instanceof Map) {
for (ext in ioptions.publish_files) {
if (args.filename.endsWith(ext.key)) {
def ext_list = path_list.collect()
ext_list.add(ext.value)
return "${getPathFromList(ext_list)}/$args.filename"
}
}
} else if (ioptions.publish_files == null) {
return "${getPathFromList(path_list)}/$args.filename"
}
}

View file

@ -0,0 +1,46 @@
include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions'
params.options = [:]
options = initOptions(params.options)
process DASTOOL_SCAFFOLDS2BIN {
tag "$meta.id"
label 'process_low'
publishDir "${params.outdir}",
mode: params.publish_dir_mode,
saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) }
conda (params.enable_conda ? "bioconda::das_tool=1.1.3" : null)
if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
container "https://depot.galaxyproject.org/singularity/das_tool:1.1.3--r41hdfd78af_0"
} else {
container "quay.io/biocontainers/das_tool:1.1.3--r41hdfd78af_0"
}
input:
tuple val(meta), path(fasta)
val(extension)
output:
tuple val(meta), path("*.tsv"), emit: scaffolds2bin
path "versions.yml" , emit: versions
script:
def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
def file_extension = extension ? extension : "fasta"
"""
gunzip -f *.${file_extension}.gz
Fasta_to_Scaffolds2Bin.sh \\
$options.args \\
-i . \\
-e $file_extension \\
> ${prefix}.tsv
cat <<-END_VERSIONS > versions.yml
${getProcessName(task.process)}:
${getSoftwareName(task.process)}: \$( DAS_Tool --version 2>&1 | grep "DAS Tool" | sed 's/DAS Tool version //' )
END_VERSIONS
"""
}

View file

@ -0,0 +1,58 @@
name: dastool_scaffolds2bin
description: Helper script to convert a set of bins in fasta format to tabular scaffolds2bin format
keywords:
- binning
- das tool
- table
- de novo
- bins
- contigs
- assembly
- das_tool
tools:
- dastool:
description: |
DAS Tool is an automated method that integrates the results
of a flexible number of binning algorithms to calculate an optimized, non-redundant
set of bins from a single assembly.
homepage: https://github.com/cmks/DAS_Tool
documentation: https://github.com/cmks/DAS_Tool
tool_dev_url: https://github.com/cmks/DAS_Tool
doi: "10.1038/s41564-018-0171-1"
licence: ['BSD']
input:
- meta:
type: map
description: |
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- fasta:
type: file
description: Fasta of list of fasta files recommended to be gathered via with .collect() of bins
pattern: "*.{fa,fas,fasta}"
- binner:
type: val
description: Name of the binning software (optional)
- extension:
type: val
description: Fasta file extension (fa | fas | fasta | ...)
output:
- meta:
type: map
description: |
Groovy Map containing sample information
e.g. [ id:'test', single_end:false ]
- versions:
type: file
description: File containing software versions
pattern: "versions.yml"
- scaffolds2bin:
type: file
description: tabular scaffolds2bin file for DAS tool input
pattern: "*.scaffolds2bin.tsv"
authors:
- "@maxibor"

View file

@ -338,6 +338,14 @@ damageprofiler:
- modules/damageprofiler/**
- tests/modules/damageprofiler/**
dastool/dastool:
- modules/dastool/dastool/**
- tests/modules/dastool/dastool/**
dastool/scaffolds2bin:
- modules/dastool/scaffolds2bin/**
- tests/modules/dastool/scaffolds2bin/**
dedup:
- modules/dedup/**
- tests/modules/dedup/**

View file

@ -0,0 +1,33 @@
#!/usr/bin/env nextflow
nextflow.enable.dsl = 2
include { METABAT2_METABAT2 } from '../../../../modules/metabat2/metabat2/main.nf' addParams( options: [args: '--minContig 1500 --minCV 0.1 --minCVSum 0.1 --minClsSize 10 --minS 2'] )
include { METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS } from '../../../../modules/metabat2/jgisummarizebamcontigdepths/main.nf' addParams( options: [:] )
include { DASTOOL_SCAFFOLDS2BIN } from '../../../../modules/dastool/scaffolds2bin/main.nf' addParams( options: [:] )
include { DASTOOL_DASTOOL } from '../../../../modules/dastool/dastool/main.nf' addParams( options: [args: '--score_threshold 0 --debug'] )
workflow test_dastool_dastool {
input_depth = [ [ id:'test', single_end:false ], // meta map
file(params.test_data['bacteroides_fragilis']['illumina']['test1_paired_end_sorted_bam'], checkIfExists: true),
file(params.test_data['bacteroides_fragilis']['illumina']['test1_paired_end_sorted_bam_bai'], checkIfExists: true) ]
METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS ( input_depth )
Channel.fromPath(params.test_data['bacteroides_fragilis']['genome']['genome_fna_gz'], checkIfExists: true)
.map { it -> [[ id:'test', single_end:false ], it] }
.join(METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS.out.depth)
.set { input_metabat2 }
METABAT2_METABAT2 ( input_metabat2 )
DASTOOL_SCAFFOLDS2BIN ( METABAT2_METABAT2.out.fasta.collect(), "fa")
Channel.of([ [ id:'test', single_end:false ], // meta map
file(params.test_data['bacteroides_fragilis']['genome']['genome_fna_gz'], checkIfExists: true)])
.join(DASTOOL_SCAFFOLDS2BIN.out.scaffolds2bin)
.set {input_dastool}
DASTOOL_DASTOOL ( input_dastool, [], [], [] )
}

View file

@ -0,0 +1,29 @@
- name: dastool dastool test_dastool_dastool
command: nextflow run tests/modules/dastool/dastool -entry test_dastool_dastool -c tests/config/nextflow.config
tags:
- dastool
- dastool/dastool
files:
- path: output/dastool/test.seqlength
md5sum: b815a5811008c36808a59b1d0dcfab24
- path: output/dastool/test.tsv
md5sum: 6e46c0be14dded7cb13af38f54feea47
- path: output/dastool/test_DASTool.log
contains:
- 'DAS Tool run on'
- path: output/dastool/test_DASTool_scaffolds2bin.txt
md5sum: 6e46c0be14dded7cb13af38f54feea47
- path: output/dastool/test_DASTool_summary.txt
md5sum: a3efa8717b30dfada78dc5ae9a3dc396
- path: output/dastool/test_proteins.faa.archaea.scg
md5sum: e79d82eecee25821d1658ea4f082601d
- path: output/dastool/test_proteins.faa.bacteria.scg
md5sum: 8132cfb17cf398d41c036ead55c96ffe
- path: output/dastool/test_test.tsv.eval
md5sum: a3efa8717b30dfada78dc5ae9a3dc396
- path: output/metabat2/bins/test.1.fa.gz
md5sum: 2b297bf557cc3831b800348859331268
- path: output/metabat2/test.tsv.gz
md5sum: 619338fa5019e361d5545ce385a6961f
- path: output/metabat2/test.txt.gz
md5sum: 745a0446af6ef68b930975e9ce5a95d6

View file

@ -0,0 +1,25 @@
#!/usr/bin/env nextflow
nextflow.enable.dsl = 2
include { METABAT2_METABAT2 } from '../../../../modules/metabat2/metabat2/main.nf' addParams( options: [args: '--minContig 1500 --minCV 0.1 --minCVSum 0.1 --minClsSize 10 --minS 2'] )
include { METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS } from '../../../../modules/metabat2/jgisummarizebamcontigdepths/main.nf' addParams( options: [:] )
include { DASTOOL_SCAFFOLDS2BIN } from '../../../../modules/dastool/scaffolds2bin/main.nf' addParams( options: [:] )
workflow test_dastool_scaffolds2bin {
input_depth = [ [ id:'test', single_end:false ], // meta map
file(params.test_data['bacteroides_fragilis']['illumina']['test1_paired_end_sorted_bam'], checkIfExists: true),
file(params.test_data['bacteroides_fragilis']['illumina']['test1_paired_end_sorted_bam_bai'], checkIfExists: true) ]
METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS ( input_depth )
Channel.fromPath(params.test_data['bacteroides_fragilis']['genome']['genome_fna_gz'], checkIfExists: true)
.map { it -> [[ id:'test', single_end:false ], it] }
.join(METABAT2_JGISUMMARIZEBAMCONTIGDEPTHS.out.depth)
.set { input_metabat2 }
METABAT2_METABAT2 ( input_metabat2 )
DASTOOL_SCAFFOLDS2BIN ( METABAT2_METABAT2.out.fasta.collect(), "fa")
}

View file

@ -0,0 +1,14 @@
- name: dastool scaffolds2bin test_dastool_scaffolds2bin
command: nextflow run tests/modules/dastool/scaffolds2bin -entry test_dastool_scaffolds2bin -c tests/config/nextflow.config
tags:
- dastool
- dastool/scaffolds2bin
files:
- path: output/dastool/test.tsv
md5sum: 6e46c0be14dded7cb13af38f54feea47
- path: output/metabat2/bins/test.1.fa.gz
md5sum: 2b297bf557cc3831b800348859331268
- path: output/metabat2/test.tsv.gz
md5sum: 619338fa5019e361d5545ce385a6961f
- path: output/metabat2/test.txt.gz
md5sum: 745a0446af6ef68b930975e9ce5a95d6