From 035443b6e092321b65574ab09620c731ed6975aa Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 23 Feb 2021 19:05:26 +0100 Subject: [PATCH 1/6] Add bcftools-mpileup module --- software/bcftools/mpileup/functions.nf | 61 +++++++++++++++++++++++ software/bcftools/mpileup/main.nf | 46 +++++++++++++++++ software/bcftools/mpileup/meta.yml | 69 ++++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 software/bcftools/mpileup/functions.nf create mode 100644 software/bcftools/mpileup/main.nf create mode 100644 software/bcftools/mpileup/meta.yml diff --git a/software/bcftools/mpileup/functions.nf b/software/bcftools/mpileup/functions.nf new file mode 100644 index 00000000..779feec6 --- /dev/null +++ b/software/bcftools/mpileup/functions.nf @@ -0,0 +1,61 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.args3 = args.args3 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/mpileup/main.nf b/software/bcftools/mpileup/main.nf new file mode 100644 index 00000000..22f32f90 --- /dev/null +++ b/software/bcftools/mpileup/main.nf @@ -0,0 +1,46 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_MPILEUP { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(bam) + path fasta + + output: + tuple val(meta), path("*.gz") , emit: vcf + tuple val(meta), path("*.tbi") , emit: tbi + tuple val(meta), path("*stats.txt"), emit: stats + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + echo "${meta.id}" > sample_name.list + bcftools mpileup \\ + --fasta-ref $fasta \\ + $options.args \\ + $bam \\ + | bcftools call --output-type v $options.args2 \\ + | bcftools reheader --samples sample_name.list \\ + | bcftools view --output-file ${prefix}.vcf.gz --output-type z $options.args3 + tabix -p vcf -f ${prefix}.vcf.gz + bcftools stats ${prefix}.vcf.gz > ${prefix}.bcftools_stats.txt + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/mpileup/meta.yml b/software/bcftools/mpileup/meta.yml new file mode 100644 index 00000000..4f137d5f --- /dev/null +++ b/software/bcftools/mpileup/meta.yml @@ -0,0 +1,69 @@ +name: bcftools_mpileup +description: Compresses VCF files +keywords: + - variant calling + - mpileup + - VCF +tools: + - consensus: + description: | + Generates genotype likelihoods at each genomic position with coverage. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: Input BAM file + pattern: "*.{bam}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: FASTA reference consensus file + pattern: "*.{fasta,fa}" + - tbi: + type: file + description: tabix index file + pattern: "*.{tbi}" + - stats: + type: file + description: Text output file containing stats + pattern: "*{stats.txt}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From 4b3c86ed9f428f97550a04cf035db92aa018abcd Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 23 Feb 2021 19:05:57 +0100 Subject: [PATCH 2/6] Add tests for bcftools-mpileup --- .github/filters.yml | 6 +++++- tests/software/bcftools/mpileup/main.nf | 16 ++++++++++++++++ tests/software/bcftools/mpileup/test.yml | 12 ++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/software/bcftools/mpileup/main.nf create mode 100644 tests/software/bcftools/mpileup/test.yml diff --git a/.github/filters.yml b/.github/filters.yml index d8d63681..e726ed60 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -18,7 +18,11 @@ bcftools_merge: - software/bcftools/merge/** - tests/software/bcftools/merge/** -bcftools_stats: +bcftools_mpileup: + - software/bcftools/mpileup/** + - tests/software/bcftools/mpileup/** + + bcftools_stats: - software/bcftools/stats/** - tests/software/bcftools/stats/** diff --git a/tests/software/bcftools/mpileup/main.nf b/tests/software/bcftools/mpileup/main.nf new file mode 100644 index 00000000..82bc3b23 --- /dev/null +++ b/tests/software/bcftools/mpileup/main.nf @@ -0,0 +1,16 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_MPILEUP } from '../../../../software/bcftools/mpileup/main.nf' addParams( options: ['args2': '--no-version --ploidy 1 --multiallelic-caller'] ) + +workflow test_bcftools_mpileup { + + def input = [] + input = [ [ id:'test' ], // meta map + [ file("${launchDir}/tests/data/bam/test-sc2-artic-v3.bam", checkIfExists: true) ]] + fasta = [ file("${launchDir}/tests/data/fasta/sarscov2/MN908947.3.fa", checkIfExists: true) ] + + BCFTOOLS_MPILEUP ( input, fasta ) + +} diff --git a/tests/software/bcftools/mpileup/test.yml b/tests/software/bcftools/mpileup/test.yml new file mode 100644 index 00000000..0f42b685 --- /dev/null +++ b/tests/software/bcftools/mpileup/test.yml @@ -0,0 +1,12 @@ +- name: bcftools mpileup + command: nextflow run ./tests/software/bcftools/mpileup -entry test_bcftools_mpileup -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_mpileup + files: + - path: output/bcftools/test.vcf.gz + md5sum: cad164d4a83dc8fa0889198acc196ce0 + - path: output/bcftools/test.vcf.gz.tbi + md5sum: 51f4c3c6eec17d54e8f793ba5ba7b350 + - path: output/bcftools/test.bcftools_stats.txt + md5sum: e632d29778c2b95b8b6e28ffec6fe7dd From 20bcb00ea085c869bfb52f0f39c690719dc8f757 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 23 Feb 2021 19:08:56 +0100 Subject: [PATCH 3/6] Fix indentation --- .github/filters.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/filters.yml b/.github/filters.yml index e726ed60..6fd11e37 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -22,7 +22,7 @@ bcftools_mpileup: - software/bcftools/mpileup/** - tests/software/bcftools/mpileup/** - bcftools_stats: +bcftools_stats: - software/bcftools/stats/** - tests/software/bcftools/stats/** From e9d90007c5f211998903e3036a4f1a953f89d503 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 23 Feb 2021 19:39:24 +0100 Subject: [PATCH 4/6] Adding arguments to make md5 hashes stable --- tests/software/bcftools/mpileup/main.nf | 3 ++- tests/software/bcftools/mpileup/test.yml | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/software/bcftools/mpileup/main.nf b/tests/software/bcftools/mpileup/main.nf index 82bc3b23..1c28df55 100644 --- a/tests/software/bcftools/mpileup/main.nf +++ b/tests/software/bcftools/mpileup/main.nf @@ -2,7 +2,8 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_MPILEUP } from '../../../../software/bcftools/mpileup/main.nf' addParams( options: ['args2': '--no-version --ploidy 1 --multiallelic-caller'] ) +include { BCFTOOLS_MPILEUP } from '../../../../software/bcftools/mpileup/main.nf' addParams( options: ['args2': '--no-version --ploidy 1 --multiallelic-caller', + 'args3': '--no-version' ] ) workflow test_bcftools_mpileup { diff --git a/tests/software/bcftools/mpileup/test.yml b/tests/software/bcftools/mpileup/test.yml index 0f42b685..0c55f614 100644 --- a/tests/software/bcftools/mpileup/test.yml +++ b/tests/software/bcftools/mpileup/test.yml @@ -5,8 +5,9 @@ - bcftools_mpileup files: - path: output/bcftools/test.vcf.gz - md5sum: cad164d4a83dc8fa0889198acc196ce0 + md5sum: f25141161cb935ef7719cfb41563ffb0 - path: output/bcftools/test.vcf.gz.tbi - md5sum: 51f4c3c6eec17d54e8f793ba5ba7b350 + md5sum: 4423be9f13fa07e5eadb13b1cbe8baf7 - path: output/bcftools/test.bcftools_stats.txt - md5sum: e632d29778c2b95b8b6e28ffec6fe7dd + md5sum: b8d6b6927f7bf024acf11ba438b54fa0 + From 138be9f91842be7181b1be5c72e5b08df867e90d Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 23 Feb 2021 21:52:15 +0100 Subject: [PATCH 5/6] Adding label to the bcftools-mpileup process --- software/bcftools/mpileup/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/software/bcftools/mpileup/main.nf b/software/bcftools/mpileup/main.nf index 22f32f90..413a2cc5 100644 --- a/software/bcftools/mpileup/main.nf +++ b/software/bcftools/mpileup/main.nf @@ -6,6 +6,7 @@ def options = initOptions(params.options) process BCFTOOLS_MPILEUP { tag "$meta.id" + label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } From af7bc743bf891f60fb000eeb79c8fb20ec51f7e7 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Wed, 24 Feb 2021 09:15:18 +0100 Subject: [PATCH 6/6] Correct yml file, add fasta to input and correct vcf on output --- software/bcftools/mpileup/meta.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/software/bcftools/mpileup/meta.yml b/software/bcftools/mpileup/meta.yml index 4f137d5f..573cebb3 100644 --- a/software/bcftools/mpileup/meta.yml +++ b/software/bcftools/mpileup/meta.yml @@ -42,6 +42,10 @@ input: type: file description: Input BAM file pattern: "*.{bam}" + - fasta: + type: file + description: FASTA reference file + pattern: "*.{fasta,fa}" output: - meta: type: map @@ -50,8 +54,8 @@ output: e.g. [ id:'test', single_end:false ] - vcf: type: file - description: FASTA reference consensus file - pattern: "*.{fasta,fa}" + description: VCF gzipped output file + pattern: "*.{vcf.gz}" - tbi: type: file description: tabix index file