From 500481833fdf9b5ab8abef64f1b31c1b3ebd45eb Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Sat, 30 Jan 2021 11:22:25 +0100 Subject: [PATCH 01/23] Adding bcftools_bgzip module --- software/bcftools/bgzip/functions.nf | 60 ++++++++++++++++++++++++++++ software/bcftools/bgzip/main.nf | 34 ++++++++++++++++ software/bcftools/bgzip/meta.yml | 55 +++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 software/bcftools/bgzip/functions.nf create mode 100644 software/bcftools/bgzip/main.nf create mode 100644 software/bcftools/bgzip/meta.yml diff --git a/software/bcftools/bgzip/functions.nf b/software/bcftools/bgzip/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/bgzip/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/bgzip/main.nf b/software/bcftools/bgzip/main.nf new file mode 100644 index 00000000..cb29c2b2 --- /dev/null +++ b/software/bcftools/bgzip/main.nf @@ -0,0 +1,34 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_BGZIP { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf) + + output: + tuple val(meta), path("*.gz"), emit: vcf + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + bgzip -c $options.args $vcf > ${prefix}.vcf.gz + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/bgzip/meta.yml b/software/bcftools/bgzip/meta.yml new file mode 100644 index 00000000..b0803961 --- /dev/null +++ b/software/bcftools/bgzip/meta.yml @@ -0,0 +1,55 @@ +name: bcftools_bgzip +description: Compresses VCF files +keywords: + - variant calling + - compress + - VCF +tools: + - bgzip: + description: | + Bgzip compresses files in a similar manner to, and compatible with, gzip. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bgzip.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF text file +output: + - vcf: + type: file + description: Output compressed VCF file + pattern: "*.{vcf}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From b6ef06c6cd8117828dcab7c18729386bc4ec839d Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Sun, 31 Jan 2021 22:23:26 +0100 Subject: [PATCH 02/23] Add bcftools_consensus module --- software/bcftools/consensus/functions.nf | 60 ++++++++++++++++++++++ software/bcftools/consensus/main.nf | 37 ++++++++++++++ software/bcftools/consensus/meta.yml | 64 ++++++++++++++++++++++++ 3 files changed, 161 insertions(+) create mode 100644 software/bcftools/consensus/functions.nf create mode 100644 software/bcftools/consensus/main.nf create mode 100644 software/bcftools/consensus/meta.yml diff --git a/software/bcftools/consensus/functions.nf b/software/bcftools/consensus/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/consensus/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/consensus/main.nf b/software/bcftools/consensus/main.nf new file mode 100644 index 00000000..c74d21d0 --- /dev/null +++ b/software/bcftools/consensus/main.nf @@ -0,0 +1,37 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_CONSENSUS { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf), path(tbi), path(fasta) + + output: + tuple val(meta), path("*.fa"), emit: fasta + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + cat $fasta | bcftools consensus $vcf $options.args > ${prefix}.fa + header=\$(head -n 1 ${prefix}.fa | sed 's/>//g') + sed -i "s/\${header}/${meta.id}/g" ${prefix}.fa + + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/consensus/meta.yml b/software/bcftools/consensus/meta.yml new file mode 100644 index 00000000..4f3405f3 --- /dev/null +++ b/software/bcftools/consensus/meta.yml @@ -0,0 +1,64 @@ +name: bcftools_consensus +description: Compresses VCF files +keywords: + - variant calling + - consensus + - VCF +tools: + - consensus: + description: | + Create consensus sequence by applying VCF variants to a reference fasta file. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF file + pattern: "*.{vcf}" + - tbi: + type: file + description: tabix index file + pattern: "*.{tbi}" + - fasta: + type: file + description: FASTA reference file + pattern: "*.{fasta,fa}" +output: + - fasta: + type: file + description: FASTA reference consensus file + pattern: "*.{fasta,fa}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From eb12f4610dd0f04c55f76db9a7e4f0e6e8d4073f Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 09:28:51 +0100 Subject: [PATCH 03/23] Fixing meta ymls --- software/bcftools/bgzip/meta.yml | 71 ++++++++++++++------------- software/bcftools/consensus/meta.yml | 73 +++++++++++++++------------- 2 files changed, 77 insertions(+), 67 deletions(-) diff --git a/software/bcftools/bgzip/meta.yml b/software/bcftools/bgzip/meta.yml index b0803961..c2f9e548 100644 --- a/software/bcftools/bgzip/meta.yml +++ b/software/bcftools/bgzip/meta.yml @@ -6,50 +6,55 @@ keywords: - VCF tools: - bgzip: - description: | - Bgzip compresses files in a similar manner to, and compatible with, gzip. - homepage: http://samtools.github.io/bcftools/bcftools.html - documentation: http://www.htslib.org/doc/bgzip.html - doi: 10.1093/bioinformatics/btp352 + description: | + Bgzip compresses files in a similar manner to, and compatible with, gzip. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bgzip.html + doi: 10.1093/bioinformatics/btp352 params: - outdir: - type: string - description: | - The pipeline's output directory. By default, the module will - output files into `$params.outdir/` + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` - publish_dir_mode: - type: string - description: | - Value for the Nextflow `publishDir` mode parameter. - Available: symlink, rellink, link, copy, copyNoFollow, move. + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. - enable_conda: - type: boolean - description: | - Run the module with Conda using the software specified - via the `conda` directive + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive - singularity_pull_docker_container: - type: boolean - description: | - Instead of directly downloading Singularity images for use with Singularity, - force the workflow to pull and convert Docker containers instead. + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. input: - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - vcf: - type: file - description: VCF text file + type: file + description: VCF text file output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - vcf: - type: file - description: Output compressed VCF file - pattern: "*.{vcf}" + type: file + description: Output compressed VCF file + pattern: "*.{vcf}" - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" + type: file + description: File containing software version + pattern: "*.{version.txt}" authors: - "@joseespinosa" - "@drpatelh" diff --git a/software/bcftools/consensus/meta.yml b/software/bcftools/consensus/meta.yml index 4f3405f3..658f1ad9 100644 --- a/software/bcftools/consensus/meta.yml +++ b/software/bcftools/consensus/meta.yml @@ -6,59 +6,64 @@ keywords: - VCF tools: - consensus: - description: | - Create consensus sequence by applying VCF variants to a reference fasta file. - homepage: http://samtools.github.io/bcftools/bcftools.html - documentation: http://www.htslib.org/doc/bcftools.html - doi: 10.1093/bioinformatics/btp352 + description: | + Create consensus sequence by applying VCF variants to a reference fasta file. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 params: - outdir: - type: string - description: | - The pipeline's output directory. By default, the module will - output files into `$params.outdir/` + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` - publish_dir_mode: - type: string - description: | - Value for the Nextflow `publishDir` mode parameter. - Available: symlink, rellink, link, copy, copyNoFollow, move. + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. - enable_conda: - type: boolean - description: | - Run the module with Conda using the software specified - via the `conda` directive + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive - singularity_pull_docker_container: - type: boolean - description: | - Instead of directly downloading Singularity images for use with Singularity, - force the workflow to pull and convert Docker containers instead. + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. input: - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - vcf: - type: file - description: VCF file - pattern: "*.{vcf}" + type: file + description: VCF file + pattern: "*.{vcf}" - tbi: - type: file - description: tabix index file - pattern: "*.{tbi}" + type: file + description: tabix index file + pattern: "*.{tbi}" - fasta: type: file description: FASTA reference file pattern: "*.{fasta,fa}" output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - fasta: type: file description: FASTA reference consensus file pattern: "*.{fasta,fa}" - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" + type: file + description: File containing software version + pattern: "*.{version.txt}" authors: - "@joseespinosa" - "@drpatelh" From da7598272e16b781af183a5bf966425c206c041e Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 09:41:08 +0100 Subject: [PATCH 04/23] Adding bcftools_filter module --- software/bcftools/filter/functions.nf | 60 ++++++++++++++++++++++++++ software/bcftools/filter/main.nf | 38 +++++++++++++++++ software/bcftools/filter/meta.yml | 61 +++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 software/bcftools/filter/functions.nf create mode 100644 software/bcftools/filter/main.nf create mode 100644 software/bcftools/filter/meta.yml diff --git a/software/bcftools/filter/functions.nf b/software/bcftools/filter/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/filter/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/filter/main.nf b/software/bcftools/filter/main.nf new file mode 100644 index 00000000..c86166a5 --- /dev/null +++ b/software/bcftools/filter/main.nf @@ -0,0 +1,38 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_FILTER { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf) + + output: + tuple val(meta), path("*.gz"), emit: vcf + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + bcftools filter \\ + --output ${prefix}.vcf.gz \\ + $options.args \\ + $vcf + + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/filter/meta.yml b/software/bcftools/filter/meta.yml new file mode 100644 index 00000000..68fe9d47 --- /dev/null +++ b/software/bcftools/filter/meta.yml @@ -0,0 +1,61 @@ +name: bcftools_filter +description: Filters VCF files +keywords: + - variant calling + - filtering + - VCF +tools: + - filter: + description: | + Apply fixed-threshold filters to VCF files. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF input file + pattern: "*.{vcf}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF filtered output file + pattern: "*.{vcf}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From 0423e686a6d7afc5139929c95d553b72337438a7 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 09:46:24 +0100 Subject: [PATCH 05/23] Adding bcftools_isec module --- software/bcftools/isec/functions.nf | 60 +++++++++++++++++++++++++++ software/bcftools/isec/main.nf | 34 ++++++++++++++++ software/bcftools/isec/meta.yml | 63 +++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 software/bcftools/isec/functions.nf create mode 100644 software/bcftools/isec/main.nf create mode 100644 software/bcftools/isec/meta.yml diff --git a/software/bcftools/isec/functions.nf b/software/bcftools/isec/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/isec/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/isec/main.nf b/software/bcftools/isec/main.nf new file mode 100644 index 00000000..4b1bfaf4 --- /dev/null +++ b/software/bcftools/isec/main.nf @@ -0,0 +1,34 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_STATS { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf) + + output: + tuple val(meta), path("*.txt"), emit: stats + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + bcftools stats $options.args $vcf > ${prefix}.bcftools_stats.txt + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/isec/meta.yml b/software/bcftools/isec/meta.yml new file mode 100644 index 00000000..79213b0f --- /dev/null +++ b/software/bcftools/isec/meta.yml @@ -0,0 +1,63 @@ +name: bcftools_isec +description: Apply set operations to VCF files +keywords: + - variant calling + - intersect + - union + - complement + - VCF +tools: + - isec: + description: | + Computes intersections, unions and complements of VCF files. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF input file + pattern: "*.{vcf}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF filtered output file + pattern: "*.{vcf}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From 52b0b8094d94ec081e445c6f1d6f5b0087cfa42f Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 10:03:25 +0100 Subject: [PATCH 06/23] Adding bcftools_stats module --- software/bcftools/stats/functions.nf | 60 +++++++++++++++++++++++++++ software/bcftools/stats/main.nf | 34 +++++++++++++++ software/bcftools/stats/meta.yml | 62 ++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 software/bcftools/stats/functions.nf create mode 100644 software/bcftools/stats/main.nf create mode 100644 software/bcftools/stats/meta.yml diff --git a/software/bcftools/stats/functions.nf b/software/bcftools/stats/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/stats/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/stats/main.nf b/software/bcftools/stats/main.nf new file mode 100644 index 00000000..4b1bfaf4 --- /dev/null +++ b/software/bcftools/stats/main.nf @@ -0,0 +1,34 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_STATS { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf) + + output: + tuple val(meta), path("*.txt"), emit: stats + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + """ + bcftools stats $options.args $vcf > ${prefix}.bcftools_stats.txt + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/stats/meta.yml b/software/bcftools/stats/meta.yml new file mode 100644 index 00000000..1eca2773 --- /dev/null +++ b/software/bcftools/stats/meta.yml @@ -0,0 +1,62 @@ +name: bcftools_stats +description: Generates stats from VCF files +keywords: + - variant calling + - stats + - VCF +tools: + - stats: + description: | + Parses VCF or BCF and produces text file stats which is suitable for + machine processing and can be plotted using plot-vcfstats. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF input file + pattern: "*.{vcf}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - txt: + type: file + description: Text output file containing stats + pattern: "*.{txt}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From 0a8915ae77788a7202b1558ea957650b4f3f9cb8 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 10:07:35 +0100 Subject: [PATCH 07/23] Adding bcftools_tabix module --- software/bcftools/tabix/functions.nf | 60 ++++++++++++++++++++++++++ software/bcftools/tabix/main.nf | 33 +++++++++++++++ software/bcftools/tabix/meta.yml | 63 ++++++++++++++++++++++++++++ 3 files changed, 156 insertions(+) create mode 100644 software/bcftools/tabix/functions.nf create mode 100644 software/bcftools/tabix/main.nf create mode 100644 software/bcftools/tabix/meta.yml diff --git a/software/bcftools/tabix/functions.nf b/software/bcftools/tabix/functions.nf new file mode 100644 index 00000000..6f3b4b29 --- /dev/null +++ b/software/bcftools/tabix/functions.nf @@ -0,0 +1,60 @@ + +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", '') } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/bcftools/tabix/main.nf b/software/bcftools/tabix/main.nf new file mode 100644 index 00000000..3e5bba50 --- /dev/null +++ b/software/bcftools/tabix/main.nf @@ -0,0 +1,33 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process BCFTOOLS_TABIX { + tag "$meta.id" + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::bcftools=1.11" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/bcftools:1.11--h7c999a4_0" + } else { + container "quay.io/biocontainers/bcftools:1.11--h7c999a4_0" + } + + input: + tuple val(meta), path(vcf) + + output: + tuple val(meta), path("*.tbi"), emit: tbi + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + """ + tabix $options.args $vcf + echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt + """ +} diff --git a/software/bcftools/tabix/meta.yml b/software/bcftools/tabix/meta.yml new file mode 100644 index 00000000..51cd0749 --- /dev/null +++ b/software/bcftools/tabix/meta.yml @@ -0,0 +1,63 @@ +name: bcftools_tabix +description: Index GFF/BED/SAM/VCF file +keywords: + - index + - vcf + - bed + - sam + - gff +tools: + - stats: + description: | + Indexes a TAB-delimited genome position file. + homepage: http://samtools.github.io/bcftools/bcftools.html + documentation: http://www.htslib.org/doc/bcftools.html + doi: 10.1093/bioinformatics/btp352 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF input file + pattern: "*.{vcf,bed,sam,gff}" +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - tbi: + type: file + description: Index file + pattern: "*.{tbi}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@joseespinosa" + - "@drpatelh" From 579de3e0380faac8d29bc773b319e286069ae836 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 1 Feb 2021 21:59:39 +0100 Subject: [PATCH 08/23] bcftools_tabix generic for any TAB-delimited genome position file --- software/bcftools/tabix/main.nf | 4 ++-- software/bcftools/tabix/meta.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/software/bcftools/tabix/main.nf b/software/bcftools/tabix/main.nf index 3e5bba50..6e1b5d07 100644 --- a/software/bcftools/tabix/main.nf +++ b/software/bcftools/tabix/main.nf @@ -18,7 +18,7 @@ process BCFTOOLS_TABIX { } input: - tuple val(meta), path(vcf) + tuple val(meta), path(gz_genome_position_file) output: tuple val(meta), path("*.tbi"), emit: tbi @@ -27,7 +27,7 @@ process BCFTOOLS_TABIX { script: def software = getSoftwareName(task.process) """ - tabix $options.args $vcf + tabix $options.args $gz_genome_position_file echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt """ } diff --git a/software/bcftools/tabix/meta.yml b/software/bcftools/tabix/meta.yml index 51cd0749..ea4c9d1a 100644 --- a/software/bcftools/tabix/meta.yml +++ b/software/bcftools/tabix/meta.yml @@ -42,8 +42,8 @@ input: e.g. [ id:'test', single_end:false ] - vcf: type: file - description: VCF input file - pattern: "*.{vcf,bed,sam,gff}" + description: TAB-delimited genome position file compressed with bgzip + pattern: "*.{vcf.gz,bed.gz,sam.gz,gff.gz}" output: - meta: type: map From a3ce63c79c1d3f92b205248198028b3ef6777062 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 09:14:00 +0100 Subject: [PATCH 09/23] Adding tests for bcftools --- .github/workflows/bcftools_bgzip.yml | 40 +++++++++++++++ .github/workflows/bcftools_consensus.yml | 44 ++++++++++++++++ .github/workflows/bcftools_filter.yml | 40 +++++++++++++++ .github/workflows/bcftools_stats.yml | 40 +++++++++++++++ .github/workflows/bcftools_tabix.yml | 42 ++++++++++++++++ output/bcftools/test.bcftools_stats.txt | 64 ++++++++++++++++++++++++ tests/software/bcftools/main.nf | 64 ++++++++++++++++++++++++ tests/software/bcftools/test.yml | 48 ++++++++++++++++++ 8 files changed, 382 insertions(+) create mode 100644 .github/workflows/bcftools_bgzip.yml create mode 100644 .github/workflows/bcftools_consensus.yml create mode 100644 .github/workflows/bcftools_filter.yml create mode 100644 .github/workflows/bcftools_stats.yml create mode 100644 .github/workflows/bcftools_tabix.yml create mode 100644 output/bcftools/test.bcftools_stats.txt create mode 100644 tests/software/bcftools/main.nf create mode 100644 tests/software/bcftools/test.yml diff --git a/.github/workflows/bcftools_bgzip.yml b/.github/workflows/bcftools_bgzip.yml new file mode 100644 index 00000000..1a5e1dd7 --- /dev/null +++ b/.github/workflows/bcftools_bgzip.yml @@ -0,0 +1,40 @@ +name: bcftools_bgzip +on: + push: + paths: + - software/bcftools/bgzip/** + - .github/workflows/bcftools_bgzip.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/bgzip/** + - .github/workflows/bcftools_bgzip.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_bgzip --symlink --wt 2 diff --git a/.github/workflows/bcftools_consensus.yml b/.github/workflows/bcftools_consensus.yml new file mode 100644 index 00000000..f53aa04c --- /dev/null +++ b/.github/workflows/bcftools_consensus.yml @@ -0,0 +1,44 @@ +name: bcftools_consensus +on: + push: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - software/bcftools/consensus/** + - .github/workflows/bcftools_consensus.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - software/bcftools/consensus/** + - .github/workflows/bcftools_consensus.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_consensus --symlink --wt 2 diff --git a/.github/workflows/bcftools_filter.yml b/.github/workflows/bcftools_filter.yml new file mode 100644 index 00000000..4b45334c --- /dev/null +++ b/.github/workflows/bcftools_filter.yml @@ -0,0 +1,40 @@ +name: bcftools_filter +on: + push: + paths: + - software/bcftools/filter/** + - .github/workflows/bcftools_filter.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/filter/** + - .github/workflows/bcftools_filter.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_filter --symlink --wt 2 diff --git a/.github/workflows/bcftools_stats.yml b/.github/workflows/bcftools_stats.yml new file mode 100644 index 00000000..1fee2bab --- /dev/null +++ b/.github/workflows/bcftools_stats.yml @@ -0,0 +1,40 @@ +name: bcftools_stats +on: + push: + paths: + - software/bcftools/stats/** + - .github/workflows/bcftools_stats.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/stats/** + - .github/workflows/bcftools_stats.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_stats --symlink --wt 2 diff --git a/.github/workflows/bcftools_tabix.yml b/.github/workflows/bcftools_tabix.yml new file mode 100644 index 00000000..2a811eb2 --- /dev/null +++ b/.github/workflows/bcftools_tabix.yml @@ -0,0 +1,42 @@ +name: bcftools_tabix +on: + push: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - .github/workflows/bcftools_tabix.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - .github/workflows/bcftools_tabix.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_tabix --symlink --wt 2 diff --git a/output/bcftools/test.bcftools_stats.txt b/output/bcftools/test.bcftools_stats.txt new file mode 100644 index 00000000..e13b7ef6 --- /dev/null +++ b/output/bcftools/test.bcftools_stats.txt @@ -0,0 +1,64 @@ +# This file was produced by bcftools stats (1.11+htslib-1.11) and can be plotted using plot-vcfstats. +# The command line was: bcftools stats test.vcf +# +# Definition of sets: +# ID [2]id [3]tab-separated file names +ID 0 test.vcf +# SN, Summary numbers: +# number of records .. number of data rows in the VCF +# number of no-ALTs .. reference-only sites, ALT is either "." or identical to REF +# number of SNPs .. number of rows with a SNP +# number of MNPs .. number of rows with a MNP, such as CC>TT +# number of indels .. number of rows with an indel +# number of others .. number of rows with other type, for example a symbolic allele or +# a complex substitution, such as ACT>TCGA +# number of multiallelic sites .. number of rows with multiple alternate alleles +# number of multiallelic SNP sites .. number of rows with multiple alternate alleles, all SNPs +# +# Note that rows containing multiple types will be counted multiple times, in each +# counter. For example, a row with a SNP and an indel increments both the SNP and +# the indel counter. +# +# SN [2]id [3]key [4]value +SN 0 number of samples: 1 +SN 0 number of records: 15 +SN 0 number of no-ALTs: 2 +SN 0 number of SNPs: 7 +SN 0 number of MNPs: 0 +SN 0 number of indels: 3 +SN 0 number of others: 3 +SN 0 number of multiallelic sites: 1 +SN 0 number of multiallelic SNP sites: 1 +# TSTV, transitions/transversions: +# TSTV [2]id [3]ts [4]tv [5]ts/tv [6]ts (1st ALT) [7]tv (1st ALT) [8]ts/tv (1st ALT) +TSTV 0 3 5 0.60 3 4 0.75 +# SiS, Singleton stats: +# SiS [2]id [3]allele count [4]number of SNPs [5]number of transitions [6]number of transversions [7]number of indels [8]repeat-consistent [9]repeat-inconsistent [10]not applicable +SiS 0 1 8 3 5 3 0 0 3 +# AF, Stats by non-reference allele frequency: +# AF [2]id [3]allele frequency [4]number of SNPs [5]number of transitions [6]number of transversions [7]number of indels [8]repeat-consistent [9]repeat-inconsistent [10]not applicable +AF 0 0.000000 8 3 5 3 0 0 3 +# QUAL, Stats by quality: +# QUAL [2]id [3]Quality [4]number of SNPs [5]number of transitions (1st ALT) [6]number of transversions (1st ALT) [7]number of indels +QUAL 0 998 7 3 4 3 +# IDD, InDel distribution: +# IDD [2]id [3]length (deletions negative) [4]number of sites [5]number of genotypes [6]mean VAF +IDD 0 -2 1 0 . +IDD 0 3 1 0 . +IDD 0 8 1 0 . +# ST, Substitution types: +# ST [2]id [3]type [4]count +ST 0 A>C 1 +ST 0 A>G 0 +ST 0 A>T 0 +ST 0 C>A 2 +ST 0 C>G 0 +ST 0 C>T 1 +ST 0 G>A 1 +ST 0 G>C 0 +ST 0 G>T 0 +ST 0 T>A 2 +ST 0 T>C 1 +ST 0 T>G 0 +# DP, Depth distribution +# DP [2]id [3]bin [4]number of genotypes [5]fraction of genotypes (%) [6]number of sites [7]fraction of sites (%) diff --git a/tests/software/bcftools/main.nf b/tests/software/bcftools/main.nf new file mode 100644 index 00000000..d01cbbf7 --- /dev/null +++ b/tests/software/bcftools/main.nf @@ -0,0 +1,64 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_FILTER } from '../../../software/bcftools/filter/main.nf' addParams( options: [:] ) +include { BCFTOOLS_STATS } from '../../../software/bcftools/stats/main.nf' addParams( options: [:] ) +include { BCFTOOLS_BGZIP } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) +include { BCFTOOLS_CONSENSUS } from '../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) + +workflow test_bcftools_filter { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_FILTER ( input ) +} + +workflow test_bcftools_stats { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_STATS ( input ) +} + +workflow test_bcftools_bgzip { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input ) +} + +workflow test_bcftools_tabix { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) +} + +workflow test_bcftools_consensus { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + def fasta = [] + fasta = Channel.of ([ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.consensus.fa", checkIfExists: true) ]) + + BCFTOOLS_BGZIP ( input ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) + BCFTOOLS_CONSENSUS ( BCFTOOLS_BGZIP.out.vcf + .join( BCFTOOLS_TABIX.out.tbi, by: [0] ) + .join( fasta, by: [0] ) ) +} + + diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml new file mode 100644 index 00000000..5650e1f1 --- /dev/null +++ b/tests/software/bcftools/test.yml @@ -0,0 +1,48 @@ +- name: Run bcftools filter test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_filter -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_filter + files: + - path: output/bcftools/test.vcf.gz + md5sum: b994e60162fd97c118ddd45b08018d98 + +- name: Run bcftools stats test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_stats -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_stats + files: + - path: output/bcftools/test.bcftools_stats.txt + md5sum: abfc6a90f84e24b2cc7e92cbce06200a + +- name: Run bcftools bgzip test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_bgzip -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_bgzip + files: + - path: output/bcftools/test.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + +- name: Run bcftools tabix test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_tabix -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_tabix + files: + - path: output/bcftools/test.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + +- name: Run bcftools consensus test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_consensus -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_consensus + files: + - path: output/bcftools/test.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test.fa + md5sum: c9e7ac4537756a0b33bcf17117f9a065 From 382ea31e7e75345ff172c68a70ec432c5ca1a889 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 09:18:28 +0100 Subject: [PATCH 10/23] Adding bcftools test data --- tests/data/vcf/test.consensus.fa | 20 ++++++++++++++++++++ tests/data/vcf/test.vcf | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 tests/data/vcf/test.consensus.fa create mode 100644 tests/data/vcf/test.vcf diff --git a/tests/data/vcf/test.consensus.fa b/tests/data/vcf/test.consensus.fa new file mode 100644 index 00000000..d7688b04 --- /dev/null +++ b/tests/data/vcf/test.consensus.fa @@ -0,0 +1,20 @@ +>1:2-501 +TACcAtATgTgACAtATAAaAAAGAACATAACCTACGTATCAACTAAAGTGGTTGTTTG +cAGAAAAGGAAGACTTAAAAAGAGTCAGTACTAACCTACATAATATATACAATGTTCATT +AAATAATAAAATGAGCTCATCATACTTAGGTCATCATAAATATATCTGAAATTCACAAAT +ATTGATCAAATGGTAAAATAGACAAGTAGATTTTAATAGGTTAAACAATTACTGATTCTC +TTGAAAGAATAAATTTAATATGAGACCTATTTCATTATAATGAACTCACAAATTAGAAAC +TTCACACTGGGGGCTGGAGAGATGGCTCAGTAGTTAAGAACACTGACTGCTCTTCTGAAG +GTCCTGAGTTCAAATCCCAGCAACCACATGGTGACTTACAACCATCTGTAATGACATCTG +ATGCCCTCTGGTGTGTCTGAAGACAGCTACAGTGTACTTACATAAAATAATAAATAAATC +TTTAAAAACAAAAAAAAAGAA +>2 +gaagatcttttccttattaaggatctgaagctctgtagatttgtattctattaaacatgg +AgagattagtgattttccatattctttaagtcattttagagtaatgtgttcttaagatAa +atcagaaaaacaaaaacttgtgctttcctgtttgaaaaacaaacagctgtggggaatgGt +gtcgggacagcctttttatAaaatttttctaaataatgttgaggctttgatacgtcaaag +ttatatttcaaatggaatcacttagacctcgtttctgagtgtcaatggccatattggggA +tttgctgctgccaatgacaGcacaccctgggaatgccccaactacttactacaaagcagt +gttacatggagaagatcttcaagagtctttttgctagatctttccttggcttttgatgtg +actcctctcaataaaatccacagtaatatagtgagtggtctcctgctccaaaccagtatt +Tcagacacagttaatccagac diff --git a/tests/data/vcf/test.vcf b/tests/data/vcf/test.vcf new file mode 100644 index 00000000..88895fc7 --- /dev/null +++ b/tests/data/vcf/test.vcf @@ -0,0 +1,24 @@ +##fileformat=VCFv4.2 +##FORMAT= +##reference=file://some/path/human_g1k_v37.fasta +##INFO= +##INFO= +##ALT= +##contig= +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT NA001 +1 5 . C A . PASS . GT 0/1 +1 5 . C T . PASS . GT 0/1 +1 7 . T A . PASS . GT . +1 10 . G A . PASS . GT 0/1 +1 12 . GACA GA . PASS . GT 0/1 +1 16 . T TAAA . PASS . GT 1/1 +1 19 . A C . PASS . GT 0/1 +1 61 . C A . PASS . GT 0/1 +2 61 . agag aa . PASS . GT 0/1 +2 119 . aaa t . PASS . GT 0/1 +2 179 . g gacgtacgt . PASS . GT 0/1 +2 200 . a . PASS END=210 GT 1/0 +2 300 . a . . PASS END=310;MinDP=10 GT 0/1 +2 320 . a <*> . PASS END=330;MinDP=20 GT 0/1 +2 481 . t c,a . PASS . GT 0/2 From c25f8f0f6cedeaedc306930c04e19ccc627386c1 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 09:57:55 +0100 Subject: [PATCH 11/23] Fixing tests --- output/bcftools/test.bcftools_stats.txt | 64 ------------------------- software/bcftools/consensus/meta.yml | 6 +-- tests/software/bcftools/test.yml | 2 +- 3 files changed, 4 insertions(+), 68 deletions(-) delete mode 100644 output/bcftools/test.bcftools_stats.txt diff --git a/output/bcftools/test.bcftools_stats.txt b/output/bcftools/test.bcftools_stats.txt deleted file mode 100644 index e13b7ef6..00000000 --- a/output/bcftools/test.bcftools_stats.txt +++ /dev/null @@ -1,64 +0,0 @@ -# This file was produced by bcftools stats (1.11+htslib-1.11) and can be plotted using plot-vcfstats. -# The command line was: bcftools stats test.vcf -# -# Definition of sets: -# ID [2]id [3]tab-separated file names -ID 0 test.vcf -# SN, Summary numbers: -# number of records .. number of data rows in the VCF -# number of no-ALTs .. reference-only sites, ALT is either "." or identical to REF -# number of SNPs .. number of rows with a SNP -# number of MNPs .. number of rows with a MNP, such as CC>TT -# number of indels .. number of rows with an indel -# number of others .. number of rows with other type, for example a symbolic allele or -# a complex substitution, such as ACT>TCGA -# number of multiallelic sites .. number of rows with multiple alternate alleles -# number of multiallelic SNP sites .. number of rows with multiple alternate alleles, all SNPs -# -# Note that rows containing multiple types will be counted multiple times, in each -# counter. For example, a row with a SNP and an indel increments both the SNP and -# the indel counter. -# -# SN [2]id [3]key [4]value -SN 0 number of samples: 1 -SN 0 number of records: 15 -SN 0 number of no-ALTs: 2 -SN 0 number of SNPs: 7 -SN 0 number of MNPs: 0 -SN 0 number of indels: 3 -SN 0 number of others: 3 -SN 0 number of multiallelic sites: 1 -SN 0 number of multiallelic SNP sites: 1 -# TSTV, transitions/transversions: -# TSTV [2]id [3]ts [4]tv [5]ts/tv [6]ts (1st ALT) [7]tv (1st ALT) [8]ts/tv (1st ALT) -TSTV 0 3 5 0.60 3 4 0.75 -# SiS, Singleton stats: -# SiS [2]id [3]allele count [4]number of SNPs [5]number of transitions [6]number of transversions [7]number of indels [8]repeat-consistent [9]repeat-inconsistent [10]not applicable -SiS 0 1 8 3 5 3 0 0 3 -# AF, Stats by non-reference allele frequency: -# AF [2]id [3]allele frequency [4]number of SNPs [5]number of transitions [6]number of transversions [7]number of indels [8]repeat-consistent [9]repeat-inconsistent [10]not applicable -AF 0 0.000000 8 3 5 3 0 0 3 -# QUAL, Stats by quality: -# QUAL [2]id [3]Quality [4]number of SNPs [5]number of transitions (1st ALT) [6]number of transversions (1st ALT) [7]number of indels -QUAL 0 998 7 3 4 3 -# IDD, InDel distribution: -# IDD [2]id [3]length (deletions negative) [4]number of sites [5]number of genotypes [6]mean VAF -IDD 0 -2 1 0 . -IDD 0 3 1 0 . -IDD 0 8 1 0 . -# ST, Substitution types: -# ST [2]id [3]type [4]count -ST 0 A>C 1 -ST 0 A>G 0 -ST 0 A>T 0 -ST 0 C>A 2 -ST 0 C>G 0 -ST 0 C>T 1 -ST 0 G>A 1 -ST 0 G>C 0 -ST 0 G>T 0 -ST 0 T>A 2 -ST 0 T>C 1 -ST 0 T>G 0 -# DP, Depth distribution -# DP [2]id [3]bin [4]number of genotypes [5]fraction of genotypes (%) [6]number of sites [7]fraction of sites (%) diff --git a/software/bcftools/consensus/meta.yml b/software/bcftools/consensus/meta.yml index 658f1ad9..6cb15f11 100644 --- a/software/bcftools/consensus/meta.yml +++ b/software/bcftools/consensus/meta.yml @@ -36,8 +36,8 @@ input: - meta: type: map description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] - vcf: type: file description: VCF file @@ -46,7 +46,7 @@ input: type: file description: tabix index file pattern: "*.{tbi}" - - fasta: + - fasta: type: file description: FASTA reference file pattern: "*.{fasta,fa}" diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml index 5650e1f1..14d305d9 100644 --- a/tests/software/bcftools/test.yml +++ b/tests/software/bcftools/test.yml @@ -43,6 +43,6 @@ - path: output/bcftools/test.vcf.gz md5sum: eb75ae1f08a1884f8edc59ed423471a2 - path: output/bcftools/test.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d + md5sum: 06d52177f819730dd409157914534e8d - path: output/bcftools/test.fa md5sum: c9e7ac4537756a0b33bcf17117f9a065 From 8e4764b8edc241ca1233dc3b74b272492e0dad3b Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 10:19:34 +0100 Subject: [PATCH 12/23] bcftools_filter adding argument to test to enable md5 check --- tests/software/bcftools/main.nf | 3 ++- tests/software/bcftools/test.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/software/bcftools/main.nf b/tests/software/bcftools/main.nf index d01cbbf7..fd2b5cf2 100644 --- a/tests/software/bcftools/main.nf +++ b/tests/software/bcftools/main.nf @@ -2,7 +2,8 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_FILTER } from '../../../software/bcftools/filter/main.nf' addParams( options: [:] ) +//keep arg of bcftools_filter, otherwise md5 will change on each execution +include { BCFTOOLS_FILTER } from '../../../software/bcftools/filter/main.nf' addParams( options: ['args': '--no-version'] ) include { BCFTOOLS_STATS } from '../../../software/bcftools/stats/main.nf' addParams( options: [:] ) include { BCFTOOLS_BGZIP } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) include { BCFTOOLS_TABIX } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml index 14d305d9..cac17b55 100644 --- a/tests/software/bcftools/test.yml +++ b/tests/software/bcftools/test.yml @@ -5,7 +5,7 @@ - bcftools_filter files: - path: output/bcftools/test.vcf.gz - md5sum: b994e60162fd97c118ddd45b08018d98 + md5sum: 16947ce72a127938d881113a1e6e696b - name: Run bcftools stats test workflow command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_stats -c tests/config/nextflow.config From 3a40ec56646e12abffc71af7c7b17cc9d7095786 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 10:50:45 +0100 Subject: [PATCH 13/23] Adding bcftools_isec taking a path with several vcfs and its tests --- .github/workflows/bcftools_isec.yml | 44 +++++++++++++++++++++++++++++ software/bcftools/isec/main.nf | 15 ++++++---- software/bcftools/isec/meta.yml | 7 +++-- tests/software/bcftools/main.nf | 32 +++++++++++++++++++++ tests/software/bcftools/test.yml | 33 ++++++++++++++++++++++ 5 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/bcftools_isec.yml diff --git a/.github/workflows/bcftools_isec.yml b/.github/workflows/bcftools_isec.yml new file mode 100644 index 00000000..54231ddb --- /dev/null +++ b/.github/workflows/bcftools_isec.yml @@ -0,0 +1,44 @@ +name: bcftools_isec +on: + push: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - software/bcftools/isec/** + - .github/workflows/bcftools_isec.yml + - tests/software/bcftools/** + pull_request: + paths: + - software/bcftools/bgzip/** + - software/bcftools/tabix/** + - software/bcftools/isec/** + - .github/workflows/bcftools_tabix.yml + - tests/software/bcftools/** + +jobs: + ci_test: + runs-on: ubuntu-latest + strategy: + matrix: + nxf_version: [20.11.0-edge] + env: + NXF_ANSI_LOG: false + steps: + - uses: actions/checkout@v2 + + - name: Install Nextflow + env: + NXF_VER: ${{ matrix.nxf_version }} + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" + - name: Install dependencies + run: python -m pip install --upgrade pip pytest-workflow + + # Test the module + - run: pytest --tag bcftools_isec --symlink --wt 2 diff --git a/software/bcftools/isec/main.nf b/software/bcftools/isec/main.nf index 4b1bfaf4..2d41dbc2 100644 --- a/software/bcftools/isec/main.nf +++ b/software/bcftools/isec/main.nf @@ -4,7 +4,7 @@ include { initOptions; saveFiles; getSoftwareName } from './functions' params.options = [:] def options = initOptions(params.options) -process BCFTOOLS_STATS { +process BCFTOOLS_ISEC { tag "$meta.id" publishDir "${params.outdir}", mode: params.publish_dir_mode, @@ -18,17 +18,20 @@ process BCFTOOLS_STATS { } input: - tuple val(meta), path(vcf) + tuple val(meta), path('vcfs/*') output: - tuple val(meta), path("*.txt"), emit: stats - path "*.version.txt" , emit: version + tuple val(meta), path("${prefix}"), emit: results + path "*.version.txt" , emit: version script: def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" + prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" """ - bcftools stats $options.args $vcf > ${prefix}.bcftools_stats.txt + bcftools isec \\ + $options.args \\ + -p $prefix \\ + */*.vcf.gz echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt """ } diff --git a/software/bcftools/isec/meta.yml b/software/bcftools/isec/meta.yml index 79213b0f..05e02ce2 100644 --- a/software/bcftools/isec/meta.yml +++ b/software/bcftools/isec/meta.yml @@ -41,9 +41,10 @@ input: Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - vcf: - type: file - description: VCF input file - pattern: "*.{vcf}" + type: files + description: | + List containing 2 or more vcf files and their respective index (tbi files) + e.g. [ 'file1.vcf', 'file1.vcf.tbi', 'file2.vcf', 'file2.vcf.tbi' ] output: - meta: type: map diff --git a/tests/software/bcftools/main.nf b/tests/software/bcftools/main.nf index fd2b5cf2..d22a9a4a 100644 --- a/tests/software/bcftools/main.nf +++ b/tests/software/bcftools/main.nf @@ -8,6 +8,11 @@ include { BCFTOOLS_STATS } from '../../../software/bcftools/stats/main.nf' addPa include { BCFTOOLS_BGZIP } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) include { BCFTOOLS_TABIX } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) include { BCFTOOLS_CONSENSUS } from '../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) +include { BCFTOOLS_ISEC } from '../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z'] ) +include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP2 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX as BCFTOOLS_TABIX2 } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) +include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP3 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX as BCFTOOLS_TABIX3 } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) workflow test_bcftools_filter { @@ -62,4 +67,31 @@ workflow test_bcftools_consensus { .join( fasta, by: [0] ) ) } +workflow test_bcftools_isec { + def input = [] + input1 = [ [ id:'test1' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + input2 = [ [ id:'test2' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + input3 = [ [ id:'test3' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input1 ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) + BCFTOOLS_BGZIP2 ( input2 ) + BCFTOOLS_TABIX2 ( BCFTOOLS_BGZIP2.out.vcf ) + BCFTOOLS_BGZIP3 ( input3 ) + BCFTOOLS_TABIX3 ( BCFTOOLS_BGZIP3.out.vcf ) + BCFTOOLS_ISEC ( BCFTOOLS_BGZIP.out.vcf + .mix( BCFTOOLS_TABIX.out.tbi ) + .mix( BCFTOOLS_BGZIP2.out.vcf ) + .mix( BCFTOOLS_TABIX2.out.tbi ) + .mix( BCFTOOLS_BGZIP3.out.vcf ) + .mix( BCFTOOLS_TABIX3.out.tbi) + .map { [ it[1] ]} + .collect() + .map { [ [id: 'test'], it ] } ) +} diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml index cac17b55..31f9593c 100644 --- a/tests/software/bcftools/test.yml +++ b/tests/software/bcftools/test.yml @@ -46,3 +46,36 @@ md5sum: 06d52177f819730dd409157914534e8d - path: output/bcftools/test.fa md5sum: c9e7ac4537756a0b33bcf17117f9a065 + +- name: Run bcftools isec test workflow + command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_isec -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_isec + files: + - path: output/bcftools/test1.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test1.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test2.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test2.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test3.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test/0000.vcf.gz + md5sum: 20bd741594f456c48e1928f07f0be342 + - path: output/bcftools/test/0000.vcf.gz.tbi + md5sum: d4357a62c60df6abcb5cc806b4681dff + - path: output/bcftools/test/0001.vcf.gz + md5sum: 20bd741594f456c48e1928f07f0be342 + - path: output/bcftools/test/0001.vcf.gz.tbi + md5sum: d4357a62c60df6abcb5cc806b4681dff + - path: output/bcftools/test/0002.vcf.gz + md5sum: 20bd741594f456c48e1928f07f0be342 + - path: output/bcftools/test/0002.vcf.gz.tbi + md5sum: d4357a62c60df6abcb5cc806b4681dff + - path: md5 output/bcftools/test/README.txt + md5sum: 9a3a041e63a18e0ad41abab96e2c5f73 + - path: md5 output/bcftools/test/sites.txt + md5sum: 01bb949ed7825ecf692bf0640e363647 From e7f15feccf3775b1a53b8e76590c76a224261d40 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 11:00:02 +0100 Subject: [PATCH 14/23] Adding `--no-version` to bcftools_isec for tests md5 check --- tests/software/bcftools/main.nf | 2 +- tests/software/bcftools/test.yml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/software/bcftools/main.nf b/tests/software/bcftools/main.nf index d22a9a4a..b4397f65 100644 --- a/tests/software/bcftools/main.nf +++ b/tests/software/bcftools/main.nf @@ -8,7 +8,7 @@ include { BCFTOOLS_STATS } from '../../../software/bcftools/stats/main.nf' addPa include { BCFTOOLS_BGZIP } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) include { BCFTOOLS_TABIX } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) include { BCFTOOLS_CONSENSUS } from '../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) -include { BCFTOOLS_ISEC } from '../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z'] ) +include { BCFTOOLS_ISEC } from '../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z --no-version'] ) include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP2 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) include { BCFTOOLS_TABIX as BCFTOOLS_TABIX2 } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP3 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml index 31f9593c..ddb29817 100644 --- a/tests/software/bcftools/test.yml +++ b/tests/software/bcftools/test.yml @@ -64,18 +64,18 @@ - path: output/bcftools/test3.vcf.gz md5sum: eb75ae1f08a1884f8edc59ed423471a2 - path: output/bcftools/test/0000.vcf.gz - md5sum: 20bd741594f456c48e1928f07f0be342 + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - path: output/bcftools/test/0000.vcf.gz.tbi - md5sum: d4357a62c60df6abcb5cc806b4681dff + md5sum: 8484b151ef902e25e54f7713d46ed90e - path: output/bcftools/test/0001.vcf.gz - md5sum: 20bd741594f456c48e1928f07f0be342 + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - path: output/bcftools/test/0001.vcf.gz.tbi - md5sum: d4357a62c60df6abcb5cc806b4681dff + md5sum: 8484b151ef902e25e54f7713d46ed90e - path: output/bcftools/test/0002.vcf.gz - md5sum: 20bd741594f456c48e1928f07f0be342 + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - path: output/bcftools/test/0002.vcf.gz.tbi - md5sum: d4357a62c60df6abcb5cc806b4681dff + md5sum: 8484b151ef902e25e54f7713d46ed90e - path: md5 output/bcftools/test/README.txt - md5sum: 9a3a041e63a18e0ad41abab96e2c5f73 + md5sum: 306fcb71aafe414046a34bedd4d14304 - path: md5 output/bcftools/test/sites.txt md5sum: 01bb949ed7825ecf692bf0640e363647 From 26829497d5feec8b9d15f162dae70e2af1b634fc Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 2 Feb 2021 12:19:57 +0100 Subject: [PATCH 15/23] Fixing typo on path for md5 check --- tests/software/bcftools/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml index ddb29817..f152bff8 100644 --- a/tests/software/bcftools/test.yml +++ b/tests/software/bcftools/test.yml @@ -75,7 +75,7 @@ md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - path: output/bcftools/test/0002.vcf.gz.tbi md5sum: 8484b151ef902e25e54f7713d46ed90e - - path: md5 output/bcftools/test/README.txt + - path: output/bcftools/test/README.txt md5sum: 306fcb71aafe414046a34bedd4d14304 - - path: md5 output/bcftools/test/sites.txt + - path: output/bcftools/test/sites.txt md5sum: 01bb949ed7825ecf692bf0640e363647 From fe412797dd0b0d7fe16d26e8c5ffd70cc7f52e7d Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 15:11:36 +0100 Subject: [PATCH 16/23] Adressing general comments on PR #134 --- software/bcftools/isec/main.nf | 4 ++-- software/bcftools/isec/meta.yml | 19 ++++++++++++------- software/bcftools/stats/meta.yml | 2 +- software/bcftools/tabix/main.nf | 4 ++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/software/bcftools/isec/main.nf b/software/bcftools/isec/main.nf index 2d41dbc2..e4d4962f 100644 --- a/software/bcftools/isec/main.nf +++ b/software/bcftools/isec/main.nf @@ -18,7 +18,7 @@ process BCFTOOLS_ISEC { } input: - tuple val(meta), path('vcfs/*') + tuple val(meta), path(vcfs), path(tbis) output: tuple val(meta), path("${prefix}"), emit: results @@ -31,7 +31,7 @@ process BCFTOOLS_ISEC { bcftools isec \\ $options.args \\ -p $prefix \\ - */*.vcf.gz + *.vcf.gz echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt """ } diff --git a/software/bcftools/isec/meta.yml b/software/bcftools/isec/meta.yml index 05e02ce2..415a3546 100644 --- a/software/bcftools/isec/meta.yml +++ b/software/bcftools/isec/meta.yml @@ -40,21 +40,26 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - vcf: + - vcfs: type: files description: | - List containing 2 or more vcf files and their respective index (tbi files) - e.g. [ 'file1.vcf', 'file1.vcf.tbi', 'file2.vcf', 'file2.vcf.tbi' ] + List containing 2 or more vcf files + e.g. [ 'file1.vcf', 'file2.vcf' ] + - tbis: + type: files + description: | + List containing the tbi index files corresponding to the vcfs input files + e.g. [ 'file1.vcf.tbi', 'file2.vcf.tbi' ] output: - meta: type: map description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - vcf: - type: file - description: VCF filtered output file - pattern: "*.{vcf}" + - results: + type: directory + description: Folder containing the set operations results perform on the vcf files + pattern: "${prefix}" - version: type: file description: File containing software version diff --git a/software/bcftools/stats/meta.yml b/software/bcftools/stats/meta.yml index 1eca2773..fd8319e6 100644 --- a/software/bcftools/stats/meta.yml +++ b/software/bcftools/stats/meta.yml @@ -49,7 +49,7 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - txt: + - stats: type: file description: Text output file containing stats pattern: "*.{txt}" diff --git a/software/bcftools/tabix/main.nf b/software/bcftools/tabix/main.nf index 6e1b5d07..3e5bba50 100644 --- a/software/bcftools/tabix/main.nf +++ b/software/bcftools/tabix/main.nf @@ -18,7 +18,7 @@ process BCFTOOLS_TABIX { } input: - tuple val(meta), path(gz_genome_position_file) + tuple val(meta), path(vcf) output: tuple val(meta), path("*.tbi"), emit: tbi @@ -27,7 +27,7 @@ process BCFTOOLS_TABIX { script: def software = getSoftwareName(task.process) """ - tabix $options.args $gz_genome_position_file + tabix $options.args $vcf echo \$(bcftools --version 2>&1) | sed 's/^.*bcftools //; s/ .*\$//' > ${software}.version.txt """ } From 83cc25860deecab9d3eb59fedb982e906ba2bc5c Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 15:16:51 +0100 Subject: [PATCH 17/23] Rearranging tests to test each module separately --- .github/filters.yml | 29 +++++++ .github/workflows/bcftools_bgzip.yml | 40 --------- .github/workflows/bcftools_consensus.yml | 44 ---------- .github/workflows/bcftools_filter.yml | 40 --------- .github/workflows/bcftools_isec.yml | 44 ---------- .github/workflows/bcftools_stats.yml | 40 --------- .github/workflows/bcftools_tabix.yml | 42 ---------- tests/software/bcftools/bgzip/main.nf | 14 ++++ tests/software/bcftools/bgzip/test.yml | 8 ++ tests/software/bcftools/consensus/main.nf | 23 +++++ tests/software/bcftools/consensus/test.yml | 12 +++ tests/software/bcftools/filter/main.nf | 15 ++++ tests/software/bcftools/filter/test.yml | 8 ++ tests/software/bcftools/isec/main.nf | 44 ++++++++++ tests/software/bcftools/isec/test.yml | 32 +++++++ tests/software/bcftools/main.nf | 97 ---------------------- tests/software/bcftools/stats/main.nf | 14 ++++ tests/software/bcftools/stats/test.yml | 8 ++ tests/software/bcftools/tabix/main.nf | 16 ++++ tests/software/bcftools/tabix/test.yml | 8 ++ tests/software/bcftools/test.yml | 81 ------------------ 21 files changed, 231 insertions(+), 428 deletions(-) delete mode 100644 .github/workflows/bcftools_bgzip.yml delete mode 100644 .github/workflows/bcftools_consensus.yml delete mode 100644 .github/workflows/bcftools_filter.yml delete mode 100644 .github/workflows/bcftools_isec.yml delete mode 100644 .github/workflows/bcftools_stats.yml delete mode 100644 .github/workflows/bcftools_tabix.yml create mode 100644 tests/software/bcftools/bgzip/main.nf create mode 100644 tests/software/bcftools/bgzip/test.yml create mode 100644 tests/software/bcftools/consensus/main.nf create mode 100644 tests/software/bcftools/consensus/test.yml create mode 100644 tests/software/bcftools/filter/main.nf create mode 100644 tests/software/bcftools/filter/test.yml create mode 100644 tests/software/bcftools/isec/main.nf create mode 100644 tests/software/bcftools/isec/test.yml delete mode 100644 tests/software/bcftools/main.nf create mode 100644 tests/software/bcftools/stats/main.nf create mode 100644 tests/software/bcftools/stats/test.yml create mode 100644 tests/software/bcftools/tabix/main.nf create mode 100644 tests/software/bcftools/tabix/test.yml delete mode 100644 tests/software/bcftools/test.yml diff --git a/.github/filters.yml b/.github/filters.yml index 6a485e8c..345f73f7 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -2,6 +2,35 @@ bandage_image: - software/bandage/image/** - tests/software/bandage/image/** +bcftools_bgzip: + - software/bowtie/bgzip/** + - tests/software/bcftools/bgzip** + +bcftools_consensus: + - software/bowtie/bgzip/** + - software/bowtie/tabix/** + - software/bowtie/consesus/** + - tests/software/bcftools/consesus** + +bcftools_filter: + - software/bowtie/filter/** + - tests/software/bcftools/filter** + +bcftools_isec: + - software/bowtie/bgzip/** + - software/bowtie/tabix/** + - software/bowtie/isec/** + - tests/software/bcftools/isec** + +bcftools_stats: + - software/bowtie/stats/** + - tests/software/bcftools/stats** + +bcftools_tabix: + - software/bowtie/bgzip/** + - software/bowtie/tabix/** + - tests/software/bcftools/tabix** + bowtie_align: - software/bowtie/align/** - software/bowtie/build/** diff --git a/.github/workflows/bcftools_bgzip.yml b/.github/workflows/bcftools_bgzip.yml deleted file mode 100644 index 1a5e1dd7..00000000 --- a/.github/workflows/bcftools_bgzip.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: bcftools_bgzip -on: - push: - paths: - - software/bcftools/bgzip/** - - .github/workflows/bcftools_bgzip.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/bgzip/** - - .github/workflows/bcftools_bgzip.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_bgzip --symlink --wt 2 diff --git a/.github/workflows/bcftools_consensus.yml b/.github/workflows/bcftools_consensus.yml deleted file mode 100644 index f53aa04c..00000000 --- a/.github/workflows/bcftools_consensus.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: bcftools_consensus -on: - push: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - software/bcftools/consensus/** - - .github/workflows/bcftools_consensus.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - software/bcftools/consensus/** - - .github/workflows/bcftools_consensus.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_consensus --symlink --wt 2 diff --git a/.github/workflows/bcftools_filter.yml b/.github/workflows/bcftools_filter.yml deleted file mode 100644 index 4b45334c..00000000 --- a/.github/workflows/bcftools_filter.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: bcftools_filter -on: - push: - paths: - - software/bcftools/filter/** - - .github/workflows/bcftools_filter.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/filter/** - - .github/workflows/bcftools_filter.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_filter --symlink --wt 2 diff --git a/.github/workflows/bcftools_isec.yml b/.github/workflows/bcftools_isec.yml deleted file mode 100644 index 54231ddb..00000000 --- a/.github/workflows/bcftools_isec.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: bcftools_isec -on: - push: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - software/bcftools/isec/** - - .github/workflows/bcftools_isec.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - software/bcftools/isec/** - - .github/workflows/bcftools_tabix.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_isec --symlink --wt 2 diff --git a/.github/workflows/bcftools_stats.yml b/.github/workflows/bcftools_stats.yml deleted file mode 100644 index 1fee2bab..00000000 --- a/.github/workflows/bcftools_stats.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: bcftools_stats -on: - push: - paths: - - software/bcftools/stats/** - - .github/workflows/bcftools_stats.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/stats/** - - .github/workflows/bcftools_stats.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_stats --symlink --wt 2 diff --git a/.github/workflows/bcftools_tabix.yml b/.github/workflows/bcftools_tabix.yml deleted file mode 100644 index 2a811eb2..00000000 --- a/.github/workflows/bcftools_tabix.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: bcftools_tabix -on: - push: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - .github/workflows/bcftools_tabix.yml - - tests/software/bcftools/** - pull_request: - paths: - - software/bcftools/bgzip/** - - software/bcftools/tabix/** - - .github/workflows/bcftools_tabix.yml - - tests/software/bcftools/** - -jobs: - ci_test: - runs-on: ubuntu-latest - strategy: - matrix: - nxf_version: [20.11.0-edge] - env: - NXF_ANSI_LOG: false - steps: - - uses: actions/checkout@v2 - - - name: Install Nextflow - env: - NXF_VER: ${{ matrix.nxf_version }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" - - name: Install dependencies - run: python -m pip install --upgrade pip pytest-workflow - - # Test the module - - run: pytest --tag bcftools_tabix --symlink --wt 2 diff --git a/tests/software/bcftools/bgzip/main.nf b/tests/software/bcftools/bgzip/main.nf new file mode 100644 index 00000000..16d81fcc --- /dev/null +++ b/tests/software/bcftools/bgzip/main.nf @@ -0,0 +1,14 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) + +workflow test_bcftools_bgzip { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input ) +} diff --git a/tests/software/bcftools/bgzip/test.yml b/tests/software/bcftools/bgzip/test.yml new file mode 100644 index 00000000..17c4b4d4 --- /dev/null +++ b/tests/software/bcftools/bgzip/test.yml @@ -0,0 +1,8 @@ +- name: Run bcftools bgzip test workflow + command: nextflow run ./tests/software/bcftools/bgzip -entry test_bcftools_bgzip -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_bgzip + files: + - path: output/bcftools/test.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 diff --git a/tests/software/bcftools/consensus/main.nf b/tests/software/bcftools/consensus/main.nf new file mode 100644 index 00000000..f5c780ec --- /dev/null +++ b/tests/software/bcftools/consensus/main.nf @@ -0,0 +1,23 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX as BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) +include { BCFTOOLS_CONSENSUS as BCFTOOLS_CONSENSUS } from '../../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) + +workflow test_bcftools_consensus { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + def fasta = [] + fasta = Channel.of ([ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.consensus.fa", checkIfExists: true) ]) + + BCFTOOLS_BGZIP ( input ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) + BCFTOOLS_CONSENSUS ( BCFTOOLS_BGZIP.out.vcf + .join( BCFTOOLS_TABIX.out.tbi, by: [0] ) + .join( fasta, by: [0] ) ) +} diff --git a/tests/software/bcftools/consensus/test.yml b/tests/software/bcftools/consensus/test.yml new file mode 100644 index 00000000..e63d8635 --- /dev/null +++ b/tests/software/bcftools/consensus/test.yml @@ -0,0 +1,12 @@ +- name: Run bcftools consensus test workflow + command: nextflow run ./tests/software/bcftools/consensus -entry test_bcftools_consensus -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_consensus + files: + - path: output/bcftools/test.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test.fa + md5sum: c9e7ac4537756a0b33bcf17117f9a065 diff --git a/tests/software/bcftools/filter/main.nf b/tests/software/bcftools/filter/main.nf new file mode 100644 index 00000000..79e3ee34 --- /dev/null +++ b/tests/software/bcftools/filter/main.nf @@ -0,0 +1,15 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +//keep --no-verson argument, otherwise md5 will change on each execution +include { BCFTOOLS_FILTER } from '../../../../software/bcftools/filter/main.nf' addParams( options: ['args': '--no-version'] ) + +workflow test_bcftools_filter { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_FILTER ( input ) +} diff --git a/tests/software/bcftools/filter/test.yml b/tests/software/bcftools/filter/test.yml new file mode 100644 index 00000000..bba7562b --- /dev/null +++ b/tests/software/bcftools/filter/test.yml @@ -0,0 +1,8 @@ +- name: Run bcftools filter test workflow + command: nextflow run ./tests/software/bcftools/filter -entry test_bcftools_filter -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_filter + files: + - path: output/bcftools/test.vcf.gz + md5sum: 16947ce72a127938d881113a1e6e696b diff --git a/tests/software/bcftools/isec/main.nf b/tests/software/bcftools/isec/main.nf new file mode 100644 index 00000000..ece2bc38 --- /dev/null +++ b/tests/software/bcftools/isec/main.nf @@ -0,0 +1,44 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) +include { BCFTOOLS_ISEC } from '../../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z --no-version'] ) +include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP2 } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX as BCFTOOLS_TABIX2 } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) +include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP3 } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX as BCFTOOLS_TABIX3 } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) + +workflow test_bcftools_isec { + + def input1, input2, input3 = [] + + input1 = [ [ id:'test1' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + input2 = [ [ id:'test2' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + input3 = [ [ id:'test3' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input1 ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) + BCFTOOLS_BGZIP2 ( input2 ) + BCFTOOLS_TABIX2 ( BCFTOOLS_BGZIP2.out.vcf ) + BCFTOOLS_BGZIP3 ( input3 ) + BCFTOOLS_TABIX3 ( BCFTOOLS_BGZIP3.out.vcf ) + + vcfs = BCFTOOLS_BGZIP.out.vcf + .mix( BCFTOOLS_BGZIP2.out.vcf ) + .mix( BCFTOOLS_BGZIP3.out.vcf ) + .map { [ it[1] ]}.collect().map { [ [id: 'test'], it ] } + + tbis = BCFTOOLS_TABIX.out.tbi + .mix( BCFTOOLS_TABIX2.out.tbi ) + .mix( BCFTOOLS_TABIX3.out.tbi ) + .map { [ it[1] ]}.collect().map { [ [id: 'test'], it ] } + + BCFTOOLS_ISEC ( vcfs.join(tbis) ) +} diff --git a/tests/software/bcftools/isec/test.yml b/tests/software/bcftools/isec/test.yml new file mode 100644 index 00000000..924e1b97 --- /dev/null +++ b/tests/software/bcftools/isec/test.yml @@ -0,0 +1,32 @@ +- name: Run bcftools isec test workflow + command: nextflow run ./tests/software/bcftools/isec -entry test_bcftools_isec -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_isec + files: + - path: output/bcftools/test1.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test1.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test2.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test2.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d + - path: output/bcftools/test3.vcf.gz + md5sum: eb75ae1f08a1884f8edc59ed423471a2 + - path: output/bcftools/test/0000.vcf.gz + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d + - path: output/bcftools/test/0000.vcf.gz.tbi + md5sum: 8484b151ef902e25e54f7713d46ed90e + - path: output/bcftools/test/0001.vcf.gz + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d + - path: output/bcftools/test/0001.vcf.gz.tbi + md5sum: 8484b151ef902e25e54f7713d46ed90e + - path: output/bcftools/test/0002.vcf.gz + md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d + - path: output/bcftools/test/0002.vcf.gz.tbi + md5sum: 8484b151ef902e25e54f7713d46ed90e + - path: output/bcftools/test/README.txt + md5sum: 306fcb71aafe414046a34bedd4d14304 + - path: output/bcftools/test/sites.txt + md5sum: 01bb949ed7825ecf692bf0640e363647 diff --git a/tests/software/bcftools/main.nf b/tests/software/bcftools/main.nf deleted file mode 100644 index b4397f65..00000000 --- a/tests/software/bcftools/main.nf +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -//keep arg of bcftools_filter, otherwise md5 will change on each execution -include { BCFTOOLS_FILTER } from '../../../software/bcftools/filter/main.nf' addParams( options: ['args': '--no-version'] ) -include { BCFTOOLS_STATS } from '../../../software/bcftools/stats/main.nf' addParams( options: [:] ) -include { BCFTOOLS_BGZIP } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) -include { BCFTOOLS_CONSENSUS } from '../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) -include { BCFTOOLS_ISEC } from '../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z --no-version'] ) -include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP2 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX as BCFTOOLS_TABIX2 } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) -include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP3 } from '../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX as BCFTOOLS_TABIX3 } from '../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) - -workflow test_bcftools_filter { - - def input = [] - input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_FILTER ( input ) -} - -workflow test_bcftools_stats { - - def input = [] - input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_STATS ( input ) -} - -workflow test_bcftools_bgzip { - - def input = [] - input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_BGZIP ( input ) -} - -workflow test_bcftools_tabix { - - def input = [] - input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_BGZIP ( input ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) -} - -workflow test_bcftools_consensus { - - def input = [] - input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - def fasta = [] - fasta = Channel.of ([ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.consensus.fa", checkIfExists: true) ]) - - BCFTOOLS_BGZIP ( input ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) - BCFTOOLS_CONSENSUS ( BCFTOOLS_BGZIP.out.vcf - .join( BCFTOOLS_TABIX.out.tbi, by: [0] ) - .join( fasta, by: [0] ) ) -} - -workflow test_bcftools_isec { - - def input = [] - input1 = [ [ id:'test1' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - input2 = [ [ id:'test2' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - input3 = [ [ id:'test3' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_BGZIP ( input1 ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) - BCFTOOLS_BGZIP2 ( input2 ) - BCFTOOLS_TABIX2 ( BCFTOOLS_BGZIP2.out.vcf ) - BCFTOOLS_BGZIP3 ( input3 ) - BCFTOOLS_TABIX3 ( BCFTOOLS_BGZIP3.out.vcf ) - BCFTOOLS_ISEC ( BCFTOOLS_BGZIP.out.vcf - .mix( BCFTOOLS_TABIX.out.tbi ) - .mix( BCFTOOLS_BGZIP2.out.vcf ) - .mix( BCFTOOLS_TABIX2.out.tbi ) - .mix( BCFTOOLS_BGZIP3.out.vcf ) - .mix( BCFTOOLS_TABIX3.out.tbi) - .map { [ it[1] ]} - .collect() - .map { [ [id: 'test'], it ] } ) -} diff --git a/tests/software/bcftools/stats/main.nf b/tests/software/bcftools/stats/main.nf new file mode 100644 index 00000000..bc2ab869 --- /dev/null +++ b/tests/software/bcftools/stats/main.nf @@ -0,0 +1,14 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_STATS } from '../../../../software/bcftools/stats/main.nf' addParams( options: [:] ) + +workflow test_bcftools_stats { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_STATS ( input ) +} diff --git a/tests/software/bcftools/stats/test.yml b/tests/software/bcftools/stats/test.yml new file mode 100644 index 00000000..f7406a6b --- /dev/null +++ b/tests/software/bcftools/stats/test.yml @@ -0,0 +1,8 @@ +- name: Run bcftools bgzip test workflow + command: nextflow run ./tests/software/bcftools/stats -entry test_bcftools_stats -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_stats + files: + - path: output/bcftools/test.bcftools_stats.txt + md5sum: abfc6a90f84e24b2cc7e92cbce06200a diff --git a/tests/software/bcftools/tabix/main.nf b/tests/software/bcftools/tabix/main.nf new file mode 100644 index 00000000..051f8901 --- /dev/null +++ b/tests/software/bcftools/tabix/main.nf @@ -0,0 +1,16 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) +include { BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) + +workflow test_bcftools_tabix { + + def input = [] + input = [ [ id:'test' ], // meta map + file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + + BCFTOOLS_BGZIP ( input ) + BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) +} diff --git a/tests/software/bcftools/tabix/test.yml b/tests/software/bcftools/tabix/test.yml new file mode 100644 index 00000000..e378f316 --- /dev/null +++ b/tests/software/bcftools/tabix/test.yml @@ -0,0 +1,8 @@ +- name: Run bcftools tabix test workflow + command: nextflow run ./tests/software/bcftools/tabix -entry test_bcftools_tabix -c tests/config/nextflow.config + tags: + - bcftools + - bcftools_tabix + files: + - path: output/bcftools/test.vcf.gz.tbi + md5sum: 06d52177f819730dd409157914534e8d diff --git a/tests/software/bcftools/test.yml b/tests/software/bcftools/test.yml deleted file mode 100644 index f152bff8..00000000 --- a/tests/software/bcftools/test.yml +++ /dev/null @@ -1,81 +0,0 @@ -- name: Run bcftools filter test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_filter -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_filter - files: - - path: output/bcftools/test.vcf.gz - md5sum: 16947ce72a127938d881113a1e6e696b - -- name: Run bcftools stats test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_stats -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_stats - files: - - path: output/bcftools/test.bcftools_stats.txt - md5sum: abfc6a90f84e24b2cc7e92cbce06200a - -- name: Run bcftools bgzip test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_bgzip -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_bgzip - files: - - path: output/bcftools/test.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - -- name: Run bcftools tabix test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_tabix -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_tabix - files: - - path: output/bcftools/test.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - -- name: Run bcftools consensus test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_consensus -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_consensus - files: - - path: output/bcftools/test.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - - path: output/bcftools/test.fa - md5sum: c9e7ac4537756a0b33bcf17117f9a065 - -- name: Run bcftools isec test workflow - command: nextflow run ./tests/software/bcftools/ -profile docker -entry test_bcftools_isec -c tests/config/nextflow.config - tags: - - bcftools - - bcftools_isec - files: - - path: output/bcftools/test1.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test1.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - - path: output/bcftools/test2.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test2.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - - path: output/bcftools/test3.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test/0000.vcf.gz - md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - - path: output/bcftools/test/0000.vcf.gz.tbi - md5sum: 8484b151ef902e25e54f7713d46ed90e - - path: output/bcftools/test/0001.vcf.gz - md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - - path: output/bcftools/test/0001.vcf.gz.tbi - md5sum: 8484b151ef902e25e54f7713d46ed90e - - path: output/bcftools/test/0002.vcf.gz - md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - - path: output/bcftools/test/0002.vcf.gz.tbi - md5sum: 8484b151ef902e25e54f7713d46ed90e - - path: output/bcftools/test/README.txt - md5sum: 306fcb71aafe414046a34bedd4d14304 - - path: output/bcftools/test/sites.txt - md5sum: 01bb949ed7825ecf692bf0640e363647 From 4fa38ebb7ac957e93df6b297552e3ab71cfc6f98 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 16:51:21 +0100 Subject: [PATCH 18/23] Separate test data to break dependencies between bcftools modules tests --- .github/filters.yml | 5 --- tests/data/vcf/test.vcf.gz | Bin 0 -> 498 bytes tests/data/vcf/test.vcf.gz.tbi | Bin 0 -> 123 bytes tests/data/vcf/test2.vcf.gz | Bin 0 -> 498 bytes tests/data/vcf/test2.vcf.gz.tbi | Bin 0 -> 123 bytes tests/data/vcf/test3.vcf.gz | Bin 0 -> 498 bytes tests/data/vcf/test3.vcf.gz.tbi | Bin 0 -> 123 bytes tests/software/bcftools/bgzip/main.nf | 2 +- tests/software/bcftools/consensus/main.nf | 15 ++------ tests/software/bcftools/consensus/test.yml | 4 -- tests/software/bcftools/filter/main.nf | 2 +- tests/software/bcftools/isec/main.nf | 42 +++++---------------- tests/software/bcftools/isec/test.yml | 12 +----- tests/software/bcftools/stats/main.nf | 2 +- tests/software/bcftools/tabix/main.nf | 5 +-- 15 files changed, 19 insertions(+), 70 deletions(-) create mode 100644 tests/data/vcf/test.vcf.gz create mode 100644 tests/data/vcf/test.vcf.gz.tbi create mode 100644 tests/data/vcf/test2.vcf.gz create mode 100644 tests/data/vcf/test2.vcf.gz.tbi create mode 100644 tests/data/vcf/test3.vcf.gz create mode 100644 tests/data/vcf/test3.vcf.gz.tbi diff --git a/.github/filters.yml b/.github/filters.yml index 345f73f7..573221aa 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -7,8 +7,6 @@ bcftools_bgzip: - tests/software/bcftools/bgzip** bcftools_consensus: - - software/bowtie/bgzip/** - - software/bowtie/tabix/** - software/bowtie/consesus/** - tests/software/bcftools/consesus** @@ -17,8 +15,6 @@ bcftools_filter: - tests/software/bcftools/filter** bcftools_isec: - - software/bowtie/bgzip/** - - software/bowtie/tabix/** - software/bowtie/isec/** - tests/software/bcftools/isec** @@ -27,7 +23,6 @@ bcftools_stats: - tests/software/bcftools/stats** bcftools_tabix: - - software/bowtie/bgzip/** - software/bowtie/tabix/** - tests/software/bcftools/tabix** diff --git a/tests/data/vcf/test.vcf.gz b/tests/data/vcf/test.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..da39e1d1b6c3ff3b1cc2d06fe8819c91f05a4534 GIT binary patch literal 498 zcmVG`WmCG#Cz()n-95 z{Fp{~mbuGwwPO-PH@lt~-YUsO=|-%{B|j-%3w$?YqE=eGd(z306-+XbGihIw4&qD_bjGMxA1*P^@I~wpuLq&Z%y=iJZD{)PXq{ zDmfL(*;{bO_6`?X;Ef|$Udz4h>`$Qu7t#77rJ{3HU2QIzxBC&r^A5s}%&aVx*Fy2q zg3zT+!>Zl!ig)3}Ju*?M14g3g_ zPaa##jF4Luw(5h9h;HWJ#4-F&|EQH{NlQR!j|{CV)tc&4Cq^wnN%K;fr-wm-nU{h` zed15Ay1t

5Z_>2bv0BDDN#1!r-gDVz@!&ex+fk&UsdjWaSYsrpHfni5)v2^-86WaP6fT? H8;Ae^YwRQC literal 0 HcmV?d00001 diff --git a/tests/data/vcf/test2.vcf.gz b/tests/data/vcf/test2.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..da39e1d1b6c3ff3b1cc2d06fe8819c91f05a4534 GIT binary patch literal 498 zcmVG`WmCG#Cz()n-95 z{Fp{~mbuGwwPO-PH@lt~-YUsO=|-%{B|j-%3w$?YqE=eGd(z306-+XbGihIw4&qD_bjGMxA1*P^@I~wpuLq&Z%y=iJZD{)PXq{ zDmfL(*;{bO_6`?X;Ef|$Udz4h>`$Qu7t#77rJ{3HU2QIzxBC&r^A5s}%&aVx*Fy2q zg3zT+!>Zl!ig)3}Ju*?M14g3g_ zPaa##jF4Luw(5h9h;HWJ#4-F&|EQH{NlQR!j|{CV)tc&4Cq^wnN%K;fr-wm-nU{h` zed15Ay1t

5Z_>2bv0BDDN#1!r-gDVz@!&ex+fk&UsdjWaSYsrpHfni5)v2^-86WaP6fT? H8;Ae^YwRQC literal 0 HcmV?d00001 diff --git a/tests/data/vcf/test3.vcf.gz b/tests/data/vcf/test3.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..da39e1d1b6c3ff3b1cc2d06fe8819c91f05a4534 GIT binary patch literal 498 zcmVG`WmCG#Cz()n-95 z{Fp{~mbuGwwPO-PH@lt~-YUsO=|-%{B|j-%3w$?YqE=eGd(z306-+XbGihIw4&qD_bjGMxA1*P^@I~wpuLq&Z%y=iJZD{)PXq{ zDmfL(*;{bO_6`?X;Ef|$Udz4h>`$Qu7t#77rJ{3HU2QIzxBC&r^A5s}%&aVx*Fy2q zg3zT+!>Zl!ig)3}Ju*?M14g3g_ zPaa##jF4Luw(5h9h;HWJ#4-F&|EQH{NlQR!j|{CV)tc&4Cq^wnN%K;fr-wm-nU{h` zed15Ay1t

5Z_>2bv0BDDN#1!r-gDVz@!&ex+fk&UsdjWaSYsrpHfni5)v2^-86WaP6fT? H8;Ae^YwRQC literal 0 HcmV?d00001 diff --git a/tests/software/bcftools/bgzip/main.nf b/tests/software/bcftools/bgzip/main.nf index 16d81fcc..02fc80d2 100644 --- a/tests/software/bcftools/bgzip/main.nf +++ b/tests/software/bcftools/bgzip/main.nf @@ -8,7 +8,7 @@ workflow test_bcftools_bgzip { def input = [] input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + [ file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ]] BCFTOOLS_BGZIP ( input ) } diff --git a/tests/software/bcftools/consensus/main.nf b/tests/software/bcftools/consensus/main.nf index f5c780ec..f6da6e40 100644 --- a/tests/software/bcftools/consensus/main.nf +++ b/tests/software/bcftools/consensus/main.nf @@ -2,22 +2,15 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX as BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) include { BCFTOOLS_CONSENSUS as BCFTOOLS_CONSENSUS } from '../../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) workflow test_bcftools_consensus { def input = [] input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - def fasta = [] - fasta = Channel.of ([ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.consensus.fa", checkIfExists: true) ]) + [ file("${launchDir}/tests/data/vcf/test.vcf.gz", checkIfExists: true) ], + [ file("${launchDir}/tests/data/vcf/test.vcf.gz.tbi", checkIfExists: true) ], + [ file("${launchDir}/tests/data/vcf/test.consensus.fa", checkIfExists: true) ] ] - BCFTOOLS_BGZIP ( input ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) - BCFTOOLS_CONSENSUS ( BCFTOOLS_BGZIP.out.vcf - .join( BCFTOOLS_TABIX.out.tbi, by: [0] ) - .join( fasta, by: [0] ) ) + BCFTOOLS_CONSENSUS ( input ) } diff --git a/tests/software/bcftools/consensus/test.yml b/tests/software/bcftools/consensus/test.yml index e63d8635..764db6f4 100644 --- a/tests/software/bcftools/consensus/test.yml +++ b/tests/software/bcftools/consensus/test.yml @@ -4,9 +4,5 @@ - bcftools - bcftools_consensus files: - - path: output/bcftools/test.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - path: output/bcftools/test.fa md5sum: c9e7ac4537756a0b33bcf17117f9a065 diff --git a/tests/software/bcftools/filter/main.nf b/tests/software/bcftools/filter/main.nf index 79e3ee34..bdf21a59 100644 --- a/tests/software/bcftools/filter/main.nf +++ b/tests/software/bcftools/filter/main.nf @@ -9,7 +9,7 @@ workflow test_bcftools_filter { def input = [] input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + [ file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ]] BCFTOOLS_FILTER ( input ) } diff --git a/tests/software/bcftools/isec/main.nf b/tests/software/bcftools/isec/main.nf index ece2bc38..7d22f437 100644 --- a/tests/software/bcftools/isec/main.nf +++ b/tests/software/bcftools/isec/main.nf @@ -2,43 +2,19 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) include { BCFTOOLS_ISEC } from '../../../../software/bcftools/isec/main.nf' addParams( options: ['args': '--nfiles +2 --output-type z --no-version'] ) -include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP2 } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX as BCFTOOLS_TABIX2 } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) -include { BCFTOOLS_BGZIP as BCFTOOLS_BGZIP3 } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) -include { BCFTOOLS_TABIX as BCFTOOLS_TABIX3 } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) workflow test_bcftools_isec { - def input1, input2, input3 = [] + def input = [] + input = [ [ id:'test' ], // meta map + [ file("${launchDir}/tests/data/vcf/test.vcf.gz", checkIfExists: true), + file("${launchDir}/tests/data/vcf/test2.vcf.gz", checkIfExists: true), + file("${launchDir}/tests/data/vcf/test3.vcf.gz", checkIfExists: true)], + [ file("${launchDir}/tests/data/vcf/test.vcf.gz.tbi", checkIfExists: true), + file("${launchDir}/tests/data/vcf/test2.vcf.gz.tbi", checkIfExists: true), + file("${launchDir}/tests/data/vcf/test3.vcf.gz.tbi", checkIfExists: true) ]] - input1 = [ [ id:'test1' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + BCFTOOLS_ISEC ( input ) - input2 = [ [ id:'test2' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - input3 = [ [ id:'test3' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] - - BCFTOOLS_BGZIP ( input1 ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) - BCFTOOLS_BGZIP2 ( input2 ) - BCFTOOLS_TABIX2 ( BCFTOOLS_BGZIP2.out.vcf ) - BCFTOOLS_BGZIP3 ( input3 ) - BCFTOOLS_TABIX3 ( BCFTOOLS_BGZIP3.out.vcf ) - - vcfs = BCFTOOLS_BGZIP.out.vcf - .mix( BCFTOOLS_BGZIP2.out.vcf ) - .mix( BCFTOOLS_BGZIP3.out.vcf ) - .map { [ it[1] ]}.collect().map { [ [id: 'test'], it ] } - - tbis = BCFTOOLS_TABIX.out.tbi - .mix( BCFTOOLS_TABIX2.out.tbi ) - .mix( BCFTOOLS_TABIX3.out.tbi ) - .map { [ it[1] ]}.collect().map { [ [id: 'test'], it ] } - - BCFTOOLS_ISEC ( vcfs.join(tbis) ) } diff --git a/tests/software/bcftools/isec/test.yml b/tests/software/bcftools/isec/test.yml index 924e1b97..860846b6 100644 --- a/tests/software/bcftools/isec/test.yml +++ b/tests/software/bcftools/isec/test.yml @@ -4,16 +4,6 @@ - bcftools - bcftools_isec files: - - path: output/bcftools/test1.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test1.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - - path: output/bcftools/test2.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - - path: output/bcftools/test2.vcf.gz.tbi - md5sum: 06d52177f819730dd409157914534e8d - - path: output/bcftools/test3.vcf.gz - md5sum: eb75ae1f08a1884f8edc59ed423471a2 - path: output/bcftools/test/0000.vcf.gz md5sum: 4e84e3b6903fa44e8bd1acdeff9b265d - path: output/bcftools/test/0000.vcf.gz.tbi @@ -27,6 +17,6 @@ - path: output/bcftools/test/0002.vcf.gz.tbi md5sum: 8484b151ef902e25e54f7713d46ed90e - path: output/bcftools/test/README.txt - md5sum: 306fcb71aafe414046a34bedd4d14304 + md5sum: 63ef64134d2685bc5d50332ef20389d2 - path: output/bcftools/test/sites.txt md5sum: 01bb949ed7825ecf692bf0640e363647 diff --git a/tests/software/bcftools/stats/main.nf b/tests/software/bcftools/stats/main.nf index bc2ab869..ee7833da 100644 --- a/tests/software/bcftools/stats/main.nf +++ b/tests/software/bcftools/stats/main.nf @@ -8,7 +8,7 @@ workflow test_bcftools_stats { def input = [] input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + [ file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ]] BCFTOOLS_STATS ( input ) } diff --git a/tests/software/bcftools/tabix/main.nf b/tests/software/bcftools/tabix/main.nf index 051f8901..ec73e911 100644 --- a/tests/software/bcftools/tabix/main.nf +++ b/tests/software/bcftools/tabix/main.nf @@ -9,8 +9,7 @@ workflow test_bcftools_tabix { def input = [] input = [ [ id:'test' ], // meta map - file("${launchDir}/tests/data/vcf/test.vcf", checkIfExists: true) ] + [ file("${launchDir}/tests/data/vcf/test.vcf.gz", checkIfExists: true) ] - BCFTOOLS_BGZIP ( input ) - BCFTOOLS_TABIX ( BCFTOOLS_BGZIP.out.vcf ) + BCFTOOLS_TABIX ( input ) } From 7a0ba8584387f460f3a9d7356b2d1ae26aaa6b6b Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 18:10:50 +0100 Subject: [PATCH 19/23] Adding again bcftools modules to filters after solving conflict --- .github/filters.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/filters.yml b/.github/filters.yml index 279eaf50..db826b81 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -2,6 +2,30 @@ bandage_image: - software/bandage/image/** - tests/software/bandage/image/** +bcftools_bgzip: + - software/bowtie/bgzip/** + - tests/software/bcftools/bgzip** + +bcftools_consensus: + - software/bowtie/consesus/** + - tests/software/bcftools/consesus** + +bcftools_filter: + - software/bowtie/filter/** + - tests/software/bcftools/filter** + +bcftools_isec: + - software/bowtie/isec/** + - tests/software/bcftools/isec** + +bcftools_stats: + - software/bowtie/stats/** + - tests/software/bcftools/stats** + +bcftools_tabix: + - software/bowtie/tabix/** + - tests/software/bcftools/tabix** + bedtools_complement: - software/bedtools/complement/** - tests/software/bedtools/complement/** From f8882de45613b8afe068d9a8d6df86c35cc95854 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 21:56:53 +0100 Subject: [PATCH 20/23] Renaming test ymls just to: software tool --- tests/software/bcftools/bgzip/test.yml | 2 +- tests/software/bcftools/consensus/test.yml | 2 +- tests/software/bcftools/filter/test.yml | 2 +- tests/software/bcftools/isec/test.yml | 2 +- tests/software/bcftools/stats/test.yml | 2 +- tests/software/bcftools/tabix/main.nf | 3 +-- tests/software/bcftools/tabix/test.yml | 2 +- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/software/bcftools/bgzip/test.yml b/tests/software/bcftools/bgzip/test.yml index 17c4b4d4..b8b69258 100644 --- a/tests/software/bcftools/bgzip/test.yml +++ b/tests/software/bcftools/bgzip/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools bgzip test workflow +- name: bcftools bgzip command: nextflow run ./tests/software/bcftools/bgzip -entry test_bcftools_bgzip -c tests/config/nextflow.config tags: - bcftools diff --git a/tests/software/bcftools/consensus/test.yml b/tests/software/bcftools/consensus/test.yml index 764db6f4..26384b3e 100644 --- a/tests/software/bcftools/consensus/test.yml +++ b/tests/software/bcftools/consensus/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools consensus test workflow +- name: bcftools consensus command: nextflow run ./tests/software/bcftools/consensus -entry test_bcftools_consensus -c tests/config/nextflow.config tags: - bcftools diff --git a/tests/software/bcftools/filter/test.yml b/tests/software/bcftools/filter/test.yml index bba7562b..a47d3cd5 100644 --- a/tests/software/bcftools/filter/test.yml +++ b/tests/software/bcftools/filter/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools filter test workflow +- name: bcftools filter command: nextflow run ./tests/software/bcftools/filter -entry test_bcftools_filter -c tests/config/nextflow.config tags: - bcftools diff --git a/tests/software/bcftools/isec/test.yml b/tests/software/bcftools/isec/test.yml index 860846b6..8955521e 100644 --- a/tests/software/bcftools/isec/test.yml +++ b/tests/software/bcftools/isec/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools isec test workflow +- name: bcftools isec command: nextflow run ./tests/software/bcftools/isec -entry test_bcftools_isec -c tests/config/nextflow.config tags: - bcftools diff --git a/tests/software/bcftools/stats/test.yml b/tests/software/bcftools/stats/test.yml index f7406a6b..fdab4904 100644 --- a/tests/software/bcftools/stats/test.yml +++ b/tests/software/bcftools/stats/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools bgzip test workflow +- name: bcftools stats command: nextflow run ./tests/software/bcftools/stats -entry test_bcftools_stats -c tests/config/nextflow.config tags: - bcftools diff --git a/tests/software/bcftools/tabix/main.nf b/tests/software/bcftools/tabix/main.nf index ec73e911..8a0f8ec6 100644 --- a/tests/software/bcftools/tabix/main.nf +++ b/tests/software/bcftools/tabix/main.nf @@ -2,14 +2,13 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_BGZIP } from '../../../../software/bcftools/bgzip/main.nf' addParams( options: [:] ) include { BCFTOOLS_TABIX } from '../../../../software/bcftools/tabix/main.nf' addParams( options: [:] ) workflow test_bcftools_tabix { def input = [] input = [ [ id:'test' ], // meta map - [ file("${launchDir}/tests/data/vcf/test.vcf.gz", checkIfExists: true) ] + [ file("${launchDir}/tests/data/vcf/test.vcf.gz", checkIfExists: true) ] ] BCFTOOLS_TABIX ( input ) } diff --git a/tests/software/bcftools/tabix/test.yml b/tests/software/bcftools/tabix/test.yml index e378f316..0356b1a8 100644 --- a/tests/software/bcftools/tabix/test.yml +++ b/tests/software/bcftools/tabix/test.yml @@ -1,4 +1,4 @@ -- name: Run bcftools tabix test workflow +- name: bcftools tabix command: nextflow run ./tests/software/bcftools/tabix -entry test_bcftools_tabix -c tests/config/nextflow.config tags: - bcftools From 58d28ed896a79eb92d18d2371223f790ff361f17 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 8 Feb 2021 21:57:31 +0100 Subject: [PATCH 21/23] Fix copy/paste error :P --- .github/filters.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/filters.yml b/.github/filters.yml index db826b81..a9dbfc13 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -3,27 +3,27 @@ bandage_image: - tests/software/bandage/image/** bcftools_bgzip: - - software/bowtie/bgzip/** + - software/bcftools/bgzip/** - tests/software/bcftools/bgzip** bcftools_consensus: - - software/bowtie/consesus/** + - software/bcftools/consesus/** - tests/software/bcftools/consesus** bcftools_filter: - - software/bowtie/filter/** + - software/bcftools/filter/** - tests/software/bcftools/filter** bcftools_isec: - - software/bowtie/isec/** + - software/bcftools/isec/** - tests/software/bcftools/isec** bcftools_stats: - - software/bowtie/stats/** + - software/bcftools/stats/** - tests/software/bcftools/stats** bcftools_tabix: - - software/bowtie/tabix/** + - software/bcftools/tabix/** - tests/software/bcftools/tabix** bedtools_complement: From 0f6830f7ccbaeead80d47ee25f728dc4cfc3afda Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 8 Feb 2021 21:41:24 +0000 Subject: [PATCH 22/23] Update tests/software/bcftools/consensus/main.nf --- tests/software/bcftools/consensus/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/software/bcftools/consensus/main.nf b/tests/software/bcftools/consensus/main.nf index f6da6e40..8b9ce4fe 100644 --- a/tests/software/bcftools/consensus/main.nf +++ b/tests/software/bcftools/consensus/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { BCFTOOLS_CONSENSUS as BCFTOOLS_CONSENSUS } from '../../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) +include { BCFTOOLS_CONSENSUS } from '../../../../software/bcftools/consensus/main.nf' addParams( options: [:] ) workflow test_bcftools_consensus { From 13c1e908a39a84c565d58e2240ddca6f5b3d9bf5 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 8 Feb 2021 21:48:21 +0000 Subject: [PATCH 23/23] Update .github/filters.yml --- .github/filters.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/filters.yml b/.github/filters.yml index a9dbfc13..435fea5f 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -7,8 +7,8 @@ bcftools_bgzip: - tests/software/bcftools/bgzip** bcftools_consensus: - - software/bcftools/consesus/** - - tests/software/bcftools/consesus** + - software/bcftools/consensus/** + - tests/software/bcftools/consensus** bcftools_filter: - software/bcftools/filter/**