From 38b450d0be865af4f60dd4674a4fc47ddbf1e6e1 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 09:54:29 +0000 Subject: [PATCH 01/91] hifiasm copied from fastqc --- software/hifiasm/functions.nf | 59 ++++++++++++++++++++++++++++ software/hifiasm/main.nf | 47 +++++++++++++++++++++++ software/hifiasm/meta.yml | 72 +++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 software/hifiasm/functions.nf create mode 100644 software/hifiasm/main.nf create mode 100644 software/hifiasm/meta.yml diff --git a/software/hifiasm/functions.nf b/software/hifiasm/functions.nf new file mode 100644 index 00000000..d25eea86 --- /dev/null +++ b/software/hifiasm/functions.nf @@ -0,0 +1,59 @@ +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf new file mode 100644 index 00000000..4e847876 --- /dev/null +++ b/software/hifiasm/main.nf @@ -0,0 +1,47 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process FASTQC { + tag "$meta.id" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + } else { + container "quay.io/biocontainers/fastqc:0.11.9--0" + } + + input: + tuple val(meta), path(reads) + + output: + tuple val(meta), path("*.html"), emit: html + tuple val(meta), path("*.zip") , emit: zip + path "*.version.txt" , emit: version + + script: + // Add soft-links to original FastQs for consistent naming in pipeline + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" + if (meta.single_end) { + """ + [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz + fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + """ + } else { + """ + [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz + [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz + fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + """ + } +} diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml new file mode 100644 index 00000000..413aad8d --- /dev/null +++ b/software/hifiasm/meta.yml @@ -0,0 +1,72 @@ +name: fastqc +description: Run FastQC on sequenced reads +keywords: + - quality control + - qc + - adapters + - fastq +tools: + - fastqc: + description: | + FastQC gives general quality metrics about your reads. + It provides information about the quality score distribution + across your reads, the per base sequence content (%A/C/G/T). + You get information about adapter contamination and other + overrepresented sequences. + homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ + documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - html: + type: file + description: FastQC report + pattern: "*_{fastqc.html}" + - zip: + type: file + description: FastQC report archive + pattern: "*_{fastqc.zip}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@drpatelh" + - "@grst" + - "@ewels" + - "@FelixKrueger" From 2f675fbe7c60a67649dd4a02e72e2f1ee79f24d3 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 11:37:06 +0000 Subject: [PATCH 02/91] hifiasm tests init from fastqc --- tests/software/hifiasm/main.nf | 28 ++++++++++++++++++++++++++++ tests/software/hifiasm/test.yml | 19 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/software/hifiasm/main.nf create mode 100644 tests/software/hifiasm/test.yml diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf new file mode 100644 index 00000000..f10b2357 --- /dev/null +++ b/tests/software/hifiasm/main.nf @@ -0,0 +1,28 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { FASTQC } from '../../../software/fastqc/main.nf' addParams( options: [:] ) + +/* + * Test with single-end data + */ +workflow test_fastqc_single_end { + + def input = [] + input = [ [ id:'test', single_end:true ], // meta map + [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] + FASTQC ( input ) +} + +/* + * Test with paired-end data + */ +workflow test_fastqc_paired_end { + + def input = [] + input = [[id: 'test', single_end: false], // meta map + [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), + file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] + FASTQC (input) +} diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml new file mode 100644 index 00000000..0fe0ebe0 --- /dev/null +++ b/tests/software/hifiasm/test.yml @@ -0,0 +1,19 @@ +- name: fastqc single-end + command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_single_end -c tests/config/nextflow.config + tags: + - fastqc + - fastqc_single_end + files: + - path: ./output/fastqc/test_fastqc.html + - path: ./output/fastqc/test_fastqc.zip + +- name: fastqc paired-end + command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_paired_end -c tests/config/nextflow.config + tags: + - fastqc + - fastqc_paired_end + files: + - path: ./output/fastqc/test_1_fastqc.html + - path: ./output/fastqc/test_2_fastqc.html + - path: ./output/fastqc/test_2_fastqc.zip + - path: ./output/fastqc/test_1_fastqc.zip From 390c835f9fc47574654aa0bfd55556bf376e98f2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 11:54:42 +0000 Subject: [PATCH 03/91] meta.yml init; test.yml and main.nf for printing version --- software/hifiasm/meta.yml | 23 ++++++++++------------- tests/software/hifiasm/main.nf | 18 +++++++++--------- tests/software/hifiasm/test.yml | 22 +++++----------------- 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml index 413aad8d..9ef62628 100644 --- a/software/hifiasm/meta.yml +++ b/software/hifiasm/meta.yml @@ -1,20 +1,17 @@ -name: fastqc -description: Run FastQC on sequenced reads +name: hifiasm +description: Run hifiasm on Pacbio HiFi reads keywords: - - quality control - - qc - - adapters + - genome assembly + - pacbaio + - hifi + - long reads - fastq tools: - - fastqc: + - hifiasm: description: | - FastQC gives general quality metrics about your reads. - It provides information about the quality score distribution - across your reads, the per base sequence content (%A/C/G/T). - You get information about adapter contamination and other - overrepresented sequences. - homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ - documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ + hifiasm generates a genome assembly from long Pacbio HiFi reads. ...Haplotype resolution ...Using short reads. + homepage: https://github.com/chhylp123/hifiasm + documentation: https://github.com/chhylp123/hifiasm params: - outdir: type: string diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf index f10b2357..66824584 100644 --- a/tests/software/hifiasm/main.nf +++ b/tests/software/hifiasm/main.nf @@ -2,27 +2,27 @@ nextflow.enable.dsl = 2 -include { FASTQC } from '../../../software/fastqc/main.nf' addParams( options: [:] ) +include { HIFIASM } from '../../../software/hifiasm/main.nf' addParams( options: [:] ) -/* +/* * Test with single-end data */ -workflow test_fastqc_single_end { +/* workflow test_fastqc_single_end { def input = [] input = [ [ id:'test', single_end:true ], // meta map [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] FASTQC ( input ) -} +} */ /* - * Test with paired-end data + * Test version printing */ -workflow test_fastqc_paired_end { +workflow test_hifiasm_version { - def input = [] +/* def input = [] input = [[id: 'test', single_end: false], // meta map [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), - file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] - FASTQC (input) + file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] */ + HIFIASM () } diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml index 0fe0ebe0..3825b238 100644 --- a/tests/software/hifiasm/test.yml +++ b/tests/software/hifiasm/test.yml @@ -1,19 +1,7 @@ -- name: fastqc single-end - command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_single_end -c tests/config/nextflow.config +- name: hifiasm + command: nextflow run ./tests/software/hifiasm/ -entry test_hifiasm_version -c tests/config/nextflow.config tags: - - fastqc - - fastqc_single_end + - hifiasm + - hifiasm_version files: - - path: ./output/fastqc/test_fastqc.html - - path: ./output/fastqc/test_fastqc.zip - -- name: fastqc paired-end - command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_paired_end -c tests/config/nextflow.config - tags: - - fastqc - - fastqc_paired_end - files: - - path: ./output/fastqc/test_1_fastqc.html - - path: ./output/fastqc/test_2_fastqc.html - - path: ./output/fastqc/test_2_fastqc.zip - - path: ./output/fastqc/test_1_fastqc.zip + - path: ./output/hifiasm/VERSION.txt From b58233794b05b287d0887c6803c6c2f7abf10db4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 13:14:43 +0000 Subject: [PATCH 04/91] Add hifiasm version printing --- software/hifiasm/main.nf | 33 +++++++++++++++++++-------------- tests/software/hifiasm/test.yml | 4 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf index 4e847876..7a364be2 100644 --- a/software/hifiasm/main.nf +++ b/software/hifiasm/main.nf @@ -4,33 +4,34 @@ include { initOptions; saveFiles; getSoftwareName } from './functions' params.options = [:] options = initOptions(params.options) -process FASTQC { - tag "$meta.id" - label 'process_medium' +process HIFIASM { + //tag "$meta.id" + label 'process_high' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:'') } // publish_id:meta.id) } - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda (params.enable_conda ? "bioconda::hifiasm=0.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + container "https://depot.galaxyproject.org/singularity/hifiasm:0.14--h8b12597_0" } else { - container "quay.io/biocontainers/fastqc:0.11.9--0" + container "quay.io/biocontainers/hifiasm:0.14--h8b12597_0" } - input: - tuple val(meta), path(reads) +/* input: + tuple val(meta), path(reads) */ output: - tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip +/* tuple val(meta), path("*.html"), emit: html + tuple val(meta), path("*.zip") , emit: zip */ path "*.version.txt" , emit: version script: // Add soft-links to original FastQs for consistent naming in pipeline def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - if (meta.single_end) { + //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" + +/* if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz @@ -43,5 +44,9 @@ process FASTQC { fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt """ - } + } */ + + """ + hifiasm --version > ${software}.version.txt || exit 0 + """ } diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml index 3825b238..e639f099 100644 --- a/tests/software/hifiasm/test.yml +++ b/tests/software/hifiasm/test.yml @@ -1,7 +1,7 @@ - name: hifiasm - command: nextflow run ./tests/software/hifiasm/ -entry test_hifiasm_version -c tests/config/nextflow.config + command: nextflow run ./tests/software/hifiasm -entry test_hifiasm_version -c tests/config/nextflow.config tags: - hifiasm - hifiasm_version files: - - path: ./output/hifiasm/VERSION.txt + - path: ./output/hifiasm/hifiasm.version.txt From 5a270c5ba7a0b382d2ca48ce42f7399a6003fa7d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 15:55:59 +0000 Subject: [PATCH 05/91] Removed spaced on an empty line --- software/hifiasm/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf index 7a364be2..8686b6bc 100644 --- a/software/hifiasm/main.nf +++ b/software/hifiasm/main.nf @@ -30,7 +30,7 @@ process HIFIASM { // Add soft-links to original FastQs for consistent naming in pipeline def software = getSoftwareName(task.process) //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - + /* if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz From 7aaf0b72486a6948f8e93f5905e56339dbca9ee5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Sun, 2 May 2021 14:09:03 +0100 Subject: [PATCH 06/91] Reverted hifiasm from main --- software/hifiasm/functions.nf | 59 ---------------------------- software/hifiasm/main.nf | 52 ------------------------- software/hifiasm/meta.yml | 69 --------------------------------- tests/software/hifiasm/main.nf | 28 ------------- tests/software/hifiasm/test.yml | 7 ---- 5 files changed, 215 deletions(-) delete mode 100644 software/hifiasm/functions.nf delete mode 100644 software/hifiasm/main.nf delete mode 100644 software/hifiasm/meta.yml delete mode 100644 tests/software/hifiasm/main.nf delete mode 100644 tests/software/hifiasm/test.yml diff --git a/software/hifiasm/functions.nf b/software/hifiasm/functions.nf deleted file mode 100644 index d25eea86..00000000 --- a/software/hifiasm/functions.nf +++ /dev/null @@ -1,59 +0,0 @@ -/* - * ----------------------------------------------------- - * Utility functions used in nf-core DSL2 module files - * ----------------------------------------------------- - */ - -/* - * Extract name of software tool from process name using $task.process - */ -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -/* - * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules - */ -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.publish_by_id = args.publish_by_id ?: false - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -/* - * Tidy up and join elements of a list to return a path string - */ -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -/* - * Function to save/publish module results - */ -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_id) { - path_list.add(args.publish_id) - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf deleted file mode 100644 index 8686b6bc..00000000 --- a/software/hifiasm/main.nf +++ /dev/null @@ -1,52 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process HIFIASM { - //tag "$meta.id" - label 'process_high' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:'') } // publish_id:meta.id) } - - conda (params.enable_conda ? "bioconda::hifiasm=0.14" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/hifiasm:0.14--h8b12597_0" - } else { - container "quay.io/biocontainers/hifiasm:0.14--h8b12597_0" - } - -/* input: - tuple val(meta), path(reads) */ - - output: -/* tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip */ - path "*.version.txt" , emit: version - - script: - // Add soft-links to original FastQs for consistent naming in pipeline - def software = getSoftwareName(task.process) - //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - -/* if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt - """ - } */ - - """ - hifiasm --version > ${software}.version.txt || exit 0 - """ -} diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml deleted file mode 100644 index 9ef62628..00000000 --- a/software/hifiasm/meta.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: hifiasm -description: Run hifiasm on Pacbio HiFi reads -keywords: - - genome assembly - - pacbaio - - hifi - - long reads - - fastq -tools: - - hifiasm: - description: | - hifiasm generates a genome assembly from long Pacbio HiFi reads. ...Haplotype resolution ...Using short reads. - homepage: https://github.com/chhylp123/hifiasm - documentation: https://github.com/chhylp123/hifiasm -params: - - outdir: - type: string - description: | - The pipeline's output directory. By default, the module will - output files into `$params.outdir/` - - publish_dir_mode: - type: string - description: | - Value for the Nextflow `publishDir` mode parameter. - Available: symlink, rellink, link, copy, copyNoFollow, move. - - enable_conda: - type: boolean - description: | - Run the module with Conda using the software specified - via the `conda` directive - - singularity_pull_docker_container: - type: boolean - description: | - Instead of directly downloading Singularity images for use with Singularity, - force the workflow to pull and convert Docker containers instead. -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - html: - type: file - description: FastQC report - pattern: "*_{fastqc.html}" - - zip: - type: file - description: FastQC report archive - pattern: "*_{fastqc.zip}" - - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" -authors: - - "@drpatelh" - - "@grst" - - "@ewels" - - "@FelixKrueger" diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf deleted file mode 100644 index 66824584..00000000 --- a/tests/software/hifiasm/main.nf +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { HIFIASM } from '../../../software/hifiasm/main.nf' addParams( options: [:] ) - -/* - * Test with single-end data - */ -/* workflow test_fastqc_single_end { - - def input = [] - input = [ [ id:'test', single_end:true ], // meta map - [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] - FASTQC ( input ) -} */ - -/* - * Test version printing - */ -workflow test_hifiasm_version { - -/* def input = [] - input = [[id: 'test', single_end: false], // meta map - [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), - file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] */ - HIFIASM () -} diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml deleted file mode 100644 index e639f099..00000000 --- a/tests/software/hifiasm/test.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: hifiasm - command: nextflow run ./tests/software/hifiasm -entry test_hifiasm_version -c tests/config/nextflow.config - tags: - - hifiasm - - hifiasm_version - files: - - path: ./output/hifiasm/hifiasm.version.txt From 1abcc938379595db0840dc32d1681cdebc268f74 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Fri, 28 Jan 2022 09:14:47 +0000 Subject: [PATCH 07/91] Ignores version file checking if testing for error --- tests/test_versions_yml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index 2f78ab2e..19ba1cf9 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -16,7 +16,8 @@ def _get_workflow_names(): # test_config = yaml.safe_load(f.read_text()) test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) for workflow in test_config: - yield workflow["name"] + if 'exit_code' not in workflow: + yield workflow["name"] @pytest.mark.workflow(*_get_workflow_names()) From 70ea9ac9a863061e22e7b5a8dd3fe07a29ecf4c4 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Fri, 28 Jan 2022 09:27:28 +0000 Subject: [PATCH 08/91] Added comment --- tests/test_versions_yml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index 19ba1cf9..5d0bb39e 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -16,7 +16,9 @@ def _get_workflow_names(): # test_config = yaml.safe_load(f.read_text()) test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) for workflow in test_config: - if 'exit_code' not in workflow: + # https://github.com/nf-core/modules/pull/1242 - added to cover tests + # that expect an error and therefore will not generate a versions.yml + if 'exit_code' not in workflow: yield workflow["name"] From 0095bc91f7d8b7a2154f321ba8567ddd8b39bf46 Mon Sep 17 00:00:00 2001 From: "Robert A. Petit III" Date: Thu, 21 Apr 2022 18:49:42 +0000 Subject: [PATCH 09/91] add module for shigatyper --- modules/shigatyper/main.nf | 64 ++++++++++++++++++++++++ modules/shigatyper/meta.yml | 47 +++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/shigatyper/main.nf | 36 +++++++++++++ tests/modules/shigatyper/nextflow.config | 5 ++ tests/modules/shigatyper/test.yml | 30 +++++++++++ 6 files changed, 186 insertions(+) create mode 100644 modules/shigatyper/main.nf create mode 100644 modules/shigatyper/meta.yml create mode 100644 tests/modules/shigatyper/main.nf create mode 100644 tests/modules/shigatyper/nextflow.config create mode 100644 tests/modules/shigatyper/test.yml diff --git a/modules/shigatyper/main.nf b/modules/shigatyper/main.nf new file mode 100644 index 00000000..3754743f --- /dev/null +++ b/modules/shigatyper/main.nf @@ -0,0 +1,64 @@ +process SHIGATYPER { + tag "$meta.id" + label 'process_low' + + conda (params.enable_conda ? "bioconda::shigatyper=2.0.1" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/shigatyper%3A2.0.1--pyhdfd78af_0': + 'quay.io/biocontainers/shigatyper:2.0.1--pyhdfd78af_0' }" + + input: + tuple val(meta), path(reads) + + output: + tuple val(meta), path("${prefix}.tsv") , emit: tsv + tuple val(meta), path("${prefix}-hits.tsv"), emit: hits, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + + if (meta.is_ont) { + """ + shigatyper \\ + $args \\ + --SE $reads \\ + --ont \\ + --name $prefix + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + shigatyper: \$(echo \$(shigatyper --version 2>&1) | sed 's/^.*ShigaTyper //' ) + END_VERSIONS + """ + } else if (meta.single_end) { + """ + shigatyper \\ + $args \\ + --SE $reads \\ + --name $prefix + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + shigatyper: \$(echo \$(shigatyper --version 2>&1) | sed 's/^.*ShigaTyper //' ) + END_VERSIONS + """ + } else { + """ + shigatyper \\ + $args \\ + --R1 ${reads[0]} \\ + --R2 ${reads[1]} \\ + --name $prefix + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + shigatyper: \$(echo \$(shigatyper --version 2>&1) | sed 's/^.*ShigaTyper //' ) + END_VERSIONS + """ + } +} diff --git a/modules/shigatyper/meta.yml b/modules/shigatyper/meta.yml new file mode 100644 index 00000000..ebaded6b --- /dev/null +++ b/modules/shigatyper/meta.yml @@ -0,0 +1,47 @@ +name: "shigatyper" +description: Determine Shigella serotype from Illumina or Oxford Nanopore reads +keywords: + - fastq + - shigella + - serotype +tools: + - "shigatyper": + description: "Typing tool for Shigella spp. from WGS Illumina sequencing" + homepage: "https://github.com/CFSAN-Biostatistics/shigatyper" + documentation: "https://github.com/CFSAN-Biostatistics/shigatyper" + tool_dev_url: "https://github.com/CFSAN-Biostatistics/shigatyper" + doi: "10.1128/AEM.00165-19" + licence: "['Public Domain']" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false, is_ont:false ] + - reads: + type: file + description: Illumina or Nanopore FASTQ file + pattern: "*.fastq.gz" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - tsv: + type: file + description: A TSV formatted file with ShigaTyper results + pattern: "*.tsv" + - hits: + type: file + description: A TSV formatted file with individual gene hits + pattern: "*-hits.tsv" + +authors: + - "@rpetit3" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 786f87db..4d7ba4fd 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1691,6 +1691,10 @@ seqwish/induce: - modules/seqwish/induce/** - tests/modules/seqwish/induce/** +shigatyper: + - modules/shigatyper/** + - tests/modules/shigatyper/** + shovill: - modules/shovill/** - tests/modules/shovill/** diff --git a/tests/modules/shigatyper/main.nf b/tests/modules/shigatyper/main.nf new file mode 100644 index 00000000..eb4df084 --- /dev/null +++ b/tests/modules/shigatyper/main.nf @@ -0,0 +1,36 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { SHIGATYPER } from '../../../modules/shigatyper/main.nf' + +workflow test_shigatyper_pe { + + input = [ + [ id:'test', single_end:false, is_ont:false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true) ] + ] + + SHIGATYPER ( input ) +} + +workflow test_shigatyper_se { + + input = [ + [ id:'test', single_end:true, is_ont:false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) ] + ] + + SHIGATYPER ( input ) +} + +workflow test_shigatyper_ont { + + input = [ + [ id:'test', single_end:true, is_ont:true ], // meta map + [ file(params.test_data['sarscov2']['nanopore']['test_fastq_gz'], checkIfExists: true) ] + ] + + SHIGATYPER ( input ) +} diff --git a/tests/modules/shigatyper/nextflow.config b/tests/modules/shigatyper/nextflow.config new file mode 100644 index 00000000..50f50a7a --- /dev/null +++ b/tests/modules/shigatyper/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/shigatyper/test.yml b/tests/modules/shigatyper/test.yml new file mode 100644 index 00000000..69a7d408 --- /dev/null +++ b/tests/modules/shigatyper/test.yml @@ -0,0 +1,30 @@ + +- name: shigatyper test_shigatyper_pe + command: nextflow run tests/modules/shigatyper -entry test_shigatyper_pe -c tests/config/nextflow.config -c tests/modules/shigatyper/nextflow.config + tags: + - shigatyper + files: + - path: output/shigatyper/test.tsv + md5sum: 4f7d38c956993800546b9acb9881d717 + - path: output/shigatyper/versions.yml + md5sum: d8ca45ed88dfba9bc570c01e4b49773b + +- name: shigatyper test_shigatyper_se + command: nextflow run tests/modules/shigatyper -entry test_shigatyper_se -c tests/config/nextflow.config -c tests/modules/shigatyper/nextflow.config + tags: + - shigatyper + files: + - path: output/shigatyper/test.tsv + md5sum: 4f7d38c956993800546b9acb9881d717 + - path: output/shigatyper/versions.yml + md5sum: 8bbf165da5a5df3b7771a33aad197eec + +- name: shigatyper test_shigatyper_ont + command: nextflow run tests/modules/shigatyper -entry test_shigatyper_ont -c tests/config/nextflow.config -c tests/modules/shigatyper/nextflow.config + tags: + - shigatyper + files: + - path: output/shigatyper/test.tsv + md5sum: 4f7d38c956993800546b9acb9881d717 + - path: output/shigatyper/versions.yml + md5sum: 0da333e1178e9e7e84a9116ad5a5ff71 From 47d8d476b864a1d0a067a57faa99ae75f9004450 Mon Sep 17 00:00:00 2001 From: "Robert A. Petit III" Date: Thu, 21 Apr 2022 12:55:56 -0600 Subject: [PATCH 10/91] make prettier --- tests/modules/shigatyper/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/modules/shigatyper/test.yml b/tests/modules/shigatyper/test.yml index 69a7d408..9dda573b 100644 --- a/tests/modules/shigatyper/test.yml +++ b/tests/modules/shigatyper/test.yml @@ -1,4 +1,3 @@ - - name: shigatyper test_shigatyper_pe command: nextflow run tests/modules/shigatyper -entry test_shigatyper_pe -c tests/config/nextflow.config -c tests/modules/shigatyper/nextflow.config tags: From c2d2399107852aed6bd8e3875cc7bf0cbc89bbb3 Mon Sep 17 00:00:00 2001 From: "Robert A. Petit III" Date: Fri, 22 Apr 2022 08:49:44 -0600 Subject: [PATCH 11/91] Update modules/shigatyper/main.nf Co-authored-by: Sateesh Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/shigatyper/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shigatyper/main.nf b/modules/shigatyper/main.nf index 3754743f..9d5c189a 100644 --- a/modules/shigatyper/main.nf +++ b/modules/shigatyper/main.nf @@ -12,7 +12,7 @@ process SHIGATYPER { output: tuple val(meta), path("${prefix}.tsv") , emit: tsv - tuple val(meta), path("${prefix}-hits.tsv"), emit: hits, optional: true + tuple val(meta), path("${prefix}-hits.tsv"), optional: true, emit: hits path "versions.yml" , emit: versions when: From 134272c7ee2de794554d51d1a55ea2a4a7e3f9a0 Mon Sep 17 00:00:00 2001 From: jasmezz Date: Tue, 26 Apr 2022 17:04:57 +0200 Subject: [PATCH 12/91] Add missing container folders as output channels --- .../antismashlitedownloaddatabases/main.nf | 6 +++++- .../antismashlitedownloaddatabases/meta.yml | 21 ++++++++++++++++--- .../antismashlitedownloaddatabases/test.yml | 11 +++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index 1853d80a..2154bafa 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -7,8 +7,9 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { 'quay.io/biocontainers/antismash-lite:6.0.1--pyhdfd78af_1' }" /* - These files are normally downloaded by download-antismash-databases itself, and must be retrieved for input by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. This is solely for use for CI tests of the nf-core/module version of antiSMASH. + These files are normally downloaded/created by download-antismash-databases itself, and must be retrieved for input by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. This is solely for use for CI tests of the nf-core/module version of antiSMASH. Reason: Upon execution, the tool checks if certain database files are present within the container and if not, it tries to create them in /usr/local/bin, for which only root user has write permissions. Mounting those database files with this module prevents the tool from trying to create them. + These files are also emitted as output channels in this module to enable the antismash-lite module to use them as mount volumes to the docker/singularity containers. */ containerOptions { @@ -26,6 +27,9 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { output: path("antismash_db") , emit: database + path("css"), emit: css_dir + path("detection"), emit: detection_dir + path("modules"), emit: modules_dir path "versions.yml", emit: versions when: diff --git a/modules/antismash/antismashlitedownloaddatabases/meta.yml b/modules/antismash/antismashlitedownloaddatabases/meta.yml index ad393bae..9e95957a 100644 --- a/modules/antismash/antismashlitedownloaddatabases/meta.yml +++ b/modules/antismash/antismashlitedownloaddatabases/meta.yml @@ -27,17 +27,17 @@ input: - database_css: type: directory description: | - antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by ther use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "css" - database_detection: type: directory description: | - antismash/detection folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/detection folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "detection" - database_modules: type: directory description: | - antismash/modules folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/modules folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "modules" output: @@ -50,6 +50,21 @@ output: type: directory description: Download directory for antiSMASH databases pattern: "antismash_db" + - css_dir: + type: directory + description: | + antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + pattern: "css" + - detection_dir: + type: directory + description: | + antismash/detection folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + pattern: "detection" + - modules_dir: + type: directory + description: | + antismash/modules folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + pattern: "modules" authors: - "@jasmezz" diff --git a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml index 3493bb4b..808e3b7e 100644 --- a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml +++ b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml @@ -1,14 +1,23 @@ - name: antismash antismashlitedownloaddatabases test_antismash_antismashlitedownloaddatabases command: nextflow run tests/modules/antismash/antismashlitedownloaddatabases -entry test_antismash_antismashlitedownloaddatabases -c tests/config/nextflow.config tags: - - antismash/antismashlitedownloaddatabases - antismash + - antismash/antismashlitedownloaddatabases files: - path: output/antismash/versions.yml md5sum: e2656c8d2bcc7469eba40eb1ee5c91b3 + - path: output/untar1/versions.yml + md5sum: b724089c7a3b22557626e3d6cf79884d + - path: output/untar2/versions.yml + md5sum: 7182a024e050e5fb6b8830930e551adc + - path: output/untar3/versions.yml + md5sum: d27a9b44dd969d74d237e52ac89bd8e5 - path: output/antismash/antismash_db - path: output/antismash/antismash_db/clusterblast - path: output/antismash/antismash_db/clustercompare - path: output/antismash/antismash_db/pfam - path: output/antismash/antismash_db/resfam - path: output/antismash/antismash_db/tigrfam + - path: output/antismash/css + - path: output/antismash/detection + - path: output/antismash/modules From d79433dcac78aa231f21177a3d69fde3cceb1cec Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Wed, 27 Apr 2022 10:08:07 +0200 Subject: [PATCH 13/91] Apply suggestions from code review Co-authored-by: James A. Fellows Yates --- modules/antismash/antismashlitedownloaddatabases/meta.yml | 2 +- .../antismash/antismashlitedownloaddatabases/test.yml | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/meta.yml b/modules/antismash/antismashlitedownloaddatabases/meta.yml index 9e95957a..619dc8c2 100644 --- a/modules/antismash/antismashlitedownloaddatabases/meta.yml +++ b/modules/antismash/antismashlitedownloaddatabases/meta.yml @@ -27,7 +27,7 @@ input: - database_css: type: directory description: | - antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by ther use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "css" - database_detection: type: directory diff --git a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml index 808e3b7e..ac38eee3 100644 --- a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml +++ b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml @@ -6,12 +6,6 @@ files: - path: output/antismash/versions.yml md5sum: e2656c8d2bcc7469eba40eb1ee5c91b3 - - path: output/untar1/versions.yml - md5sum: b724089c7a3b22557626e3d6cf79884d - - path: output/untar2/versions.yml - md5sum: 7182a024e050e5fb6b8830930e551adc - - path: output/untar3/versions.yml - md5sum: d27a9b44dd969d74d237e52ac89bd8e5 - path: output/antismash/antismash_db - path: output/antismash/antismash_db/clusterblast - path: output/antismash/antismash_db/clustercompare From 31547edc1e2491c16c00ffff4ed34de193f72b65 Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Wed, 27 Apr 2022 10:14:14 +0200 Subject: [PATCH 14/91] Update tool name --- modules/antismash/antismashlitedownloaddatabases/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index 2154bafa..72314eee 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -44,7 +44,7 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { cat <<-END_VERSIONS > versions.yml "${task.process}": - antismash: \$(antismash --version | sed 's/antiSMASH //') + antismash-lite: \$(antismash --version | sed 's/antiSMASH //') END_VERSIONS """ } From 024c992ca77d6bf5e4baddae30a38cd14d526042 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 27 Apr 2022 11:12:58 +0200 Subject: [PATCH 15/91] created the module --- modules/vardictjava/main.nf | 77 +++++++++++++++++++++++ modules/vardictjava/meta.yml | 51 +++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/vardictjava/main.nf | 15 +++++ tests/modules/vardictjava/nextflow.config | 8 +++ tests/modules/vardictjava/test.yml | 12 ++++ 6 files changed, 167 insertions(+) create mode 100644 modules/vardictjava/main.nf create mode 100644 modules/vardictjava/meta.yml create mode 100644 tests/modules/vardictjava/main.nf create mode 100644 tests/modules/vardictjava/nextflow.config create mode 100644 tests/modules/vardictjava/test.yml diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf new file mode 100644 index 00000000..d52e15dd --- /dev/null +++ b/modules/vardictjava/main.nf @@ -0,0 +1,77 @@ +// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) +// https://github.com/nf-core/modules/tree/master/modules +// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: +// https://nf-co.re/join +// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. +// All other parameters MUST be provided using the "task.ext" directive, see here: +// https://www.nextflow.io/docs/latest/process.html#ext +// where "task.ext" is a string. +// Any parameters that need to be evaluated in the context of a particular sample +// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. +// TODO nf-core: Software that can be piped together SHOULD be added to separate module files +// unless there is a run-time, storage advantage in implementing in this way +// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: +// bwa mem | samtools view -B -T ref.fasta +// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty +// list (`[]`) instead of a file can be used to work around this issue. + +process VARDICTJAVA { + tag "$meta.id" + label 'process_medium' + + // TODO nf-core: List required Conda package(s). + // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). + // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. + // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. + conda (params.enable_conda ? "bioconda::vardict-java=1.8.3" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/vardict-java:1.8.3--hdfd78af_0': + 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" + + input: + // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" + // MUST be provided as an input via a Groovy Map called "meta". + // This information may not be required in some instances e.g. indexing reference genome files: + // https://github.com/nf-core/modules/blob/master/modules/bwa/index/main.nf + // TODO nf-core: Where applicable please provide/convert compressed files as input/output + // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. + tuple val(meta), path(bam) + path(reference_fasta) + path(regions_of_interest) + + output: + // TODO nf-core: Named file extensions MUST be emitted for ALL output channels + tuple val(meta), path("*.bam"), emit: bam + // TODO nf-core: List additional required output channels/values here + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 + // If the software is unable to output a version number on the command-line then it can be manually specified + // e.g. https://github.com/nf-core/modules/blob/master/modules/homer/annotatepeaks/main.nf + // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) + // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive + // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter + // using the Nextflow "task" variable e.g. "--threads $task.cpus" + // TODO nf-core: Please replace the example samtools command below with your module's command + // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + """ + vardict-java \\ + $args \\ + -b $bam \\ + -th $task.cpus \\ + -n $prefix \\ + -G $reference_fasta \\ + $regions_of_interest \\ + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + vardictjava: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) + END_VERSIONS + """ +} diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml new file mode 100644 index 00000000..e8c55de3 --- /dev/null +++ b/modules/vardictjava/meta.yml @@ -0,0 +1,51 @@ +name: "vardictjava" +## TODO nf-core: Add a description of the module and list keywords +description: write your description here +keywords: + - sort +tools: + - "vardictjava": + ## TODO nf-core: Add a description and other details for the software below + description: "Java port of the VarDict variant discovery program" + homepage: "None" + documentation: "None" + tool_dev_url: "None" + doi: "" + licence: "['MIT']" + +## TODO nf-core: Add a description of all of the variables used as input +input: + # Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + # + ## TODO nf-core: Delete / customise this example input + - bam: + type: file + description: BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +## TODO nf-core: Add a description of all of the variables used as output +output: + #Only when we have meta + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + # + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + ## TODO nf-core: Delete / customise this example output + - bam: + type: file + description: Sorted BAM/CRAM/SAM file + pattern: "*.{bam,cram,sam}" + +authors: + - "@nvnieuwk" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 4d8ce0b5..8ca9f2e2 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1908,6 +1908,10 @@ unzip: - modules/unzip/** - tests/modules/unzip/** +vardictjava: + - modules/vardictjava/** + - tests/modules/vardictjava/** + variantbam: - modules/variantbam/** - tests/modules/variantbam/** diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf new file mode 100644 index 00000000..8a714b2b --- /dev/null +++ b/tests/modules/vardictjava/main.nf @@ -0,0 +1,15 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { VARDICTJAVA } from '../../../modules/vardictjava/main.nf' + +workflow test_vardictjava { + + input = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + + VARDICTJAVA ( input ) +} diff --git a/tests/modules/vardictjava/nextflow.config b/tests/modules/vardictjava/nextflow.config new file mode 100644 index 00000000..5dc176a9 --- /dev/null +++ b/tests/modules/vardictjava/nextflow.config @@ -0,0 +1,8 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + + withName: VARDICTJAVA { + ext.args = '' + } +} \ No newline at end of file diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml new file mode 100644 index 00000000..73c647af --- /dev/null +++ b/tests/modules/vardictjava/test.yml @@ -0,0 +1,12 @@ +## TODO nf-core: Please run the following command to build this file: +# nf-core modules create-test-yml vardictjava +- name: "vardictjava" + command: nextflow run ./tests/modules/vardictjava -entry test_vardictjava -c ./tests/config/nextflow.config -c ./tests/modules/vardictjava/nextflow.config + tags: + - "vardictjava" + # + files: + - path: "output/vardictjava/test.bam" + md5sum: e667c7caad0bc4b7ac383fd023c654fc + - path: output/vardictjava/versions.yml + md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From d57664b576d2433deed1b0fb742f3a3da05d5046 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 27 Apr 2022 11:18:58 +0200 Subject: [PATCH 16/91] cleanup TODO's --- modules/vardictjava/main.nf | 39 +------------------------------------ 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index d52e15dd..80beb540 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -1,48 +1,19 @@ -// TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) -// https://github.com/nf-core/modules/tree/master/modules -// You can also ask for help via your pull request or on the #modules channel on the nf-core Slack workspace: -// https://nf-co.re/join -// TODO nf-core: A module file SHOULD only define input and output files as command-line parameters. -// All other parameters MUST be provided using the "task.ext" directive, see here: -// https://www.nextflow.io/docs/latest/process.html#ext -// where "task.ext" is a string. -// Any parameters that need to be evaluated in the context of a particular sample -// e.g. single-end/paired-end data MUST also be defined and evaluated appropriately. -// TODO nf-core: Software that can be piped together SHOULD be added to separate module files -// unless there is a run-time, storage advantage in implementing in this way -// e.g. it's ok to have a single module for bwa to output BAM instead of SAM: -// bwa mem | samtools view -B -T ref.fasta -// TODO nf-core: Optional inputs are not currently supported by Nextflow. However, using an empty -// list (`[]`) instead of a file can be used to work around this issue. - process VARDICTJAVA { tag "$meta.id" label 'process_medium' - // TODO nf-core: List required Conda package(s). - // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). - // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. - // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. conda (params.enable_conda ? "bioconda::vardict-java=1.8.3" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/vardict-java:1.8.3--hdfd78af_0': 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - // TODO nf-core: Where applicable all sample-specific information e.g. "id", "single_end", "read_group" - // MUST be provided as an input via a Groovy Map called "meta". - // This information may not be required in some instances e.g. indexing reference genome files: - // https://github.com/nf-core/modules/blob/master/modules/bwa/index/main.nf - // TODO nf-core: Where applicable please provide/convert compressed files as input/output - // e.g. "*.fastq.gz" and NOT "*.fastq", "*.bam" and NOT "*.sam" etc. tuple val(meta), path(bam) path(reference_fasta) path(regions_of_interest) output: - // TODO nf-core: Named file extensions MUST be emitted for ALL output channels tuple val(meta), path("*.bam"), emit: bam - // TODO nf-core: List additional required output channels/values here path "versions.yml" , emit: versions when: @@ -51,15 +22,7 @@ process VARDICTJAVA { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 - // If the software is unable to output a version number on the command-line then it can be manually specified - // e.g. https://github.com/nf-core/modules/blob/master/modules/homer/annotatepeaks/main.nf - // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) - // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "task.ext.args" directive - // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter - // using the Nextflow "task" variable e.g. "--threads $task.cpus" - // TODO nf-core: Please replace the example samtools command below with your module's command - // TODO nf-core: Please indent the command appropriately (4 spaces!!) to help with readability ;) + """ vardict-java \\ $args \\ From 9108e2e2ec337978929cafc8d7cfedb656124786 Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Wed, 27 Apr 2022 12:22:05 +0200 Subject: [PATCH 17/91] Update test output checksum --- tests/modules/antismash/antismashlitedownloaddatabases/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml index ac38eee3..1079363c 100644 --- a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml +++ b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml @@ -5,7 +5,7 @@ - antismash/antismashlitedownloaddatabases files: - path: output/antismash/versions.yml - md5sum: e2656c8d2bcc7469eba40eb1ee5c91b3 + md5sum: 24859c67023abab99de295d3675a24b6 - path: output/antismash/antismash_db - path: output/antismash/antismash_db/clusterblast - path: output/antismash/antismash_db/clustercompare From 7956d38e61a8efdd2fffcb6cf33f4a02ad73bc50 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 27 Apr 2022 13:18:46 +0200 Subject: [PATCH 18/91] Fixed the test on nextflow level --- modules/vardictjava/main.nf | 4 +-- modules/vardictjava/meta.yml | 46 ++++++++++++++++++------------ tests/modules/vardictjava/main.nf | 8 ++++-- tests/modules/vardictjava/test.yml | 2 +- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 80beb540..8170bf76 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -13,7 +13,7 @@ process VARDICTJAVA { path(regions_of_interest) output: - tuple val(meta), path("*.bam"), emit: bam + tuple val(meta), path("*.vcf"), emit: vcf path "versions.yml" , emit: versions when: @@ -34,7 +34,7 @@ process VARDICTJAVA { cat <<-END_VERSIONS > versions.yml "${task.process}": - vardictjava: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) + vardictjava: \$(echo 1.8.3) END_VERSIONS """ } diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index e8c55de3..7820a877 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -1,19 +1,19 @@ name: "vardictjava" -## TODO nf-core: Add a description of the module and list keywords -description: write your description here + +description: The Java port of the VarDict variant caller keywords: - - sort + - variant calling + - VarDict + - AstraZeneca tools: - "vardictjava": - ## TODO nf-core: Add a description and other details for the software below description: "Java port of the VarDict variant discovery program" - homepage: "None" - documentation: "None" - tool_dev_url: "None" - doi: "" + homepage: "https://github.com/AstraZeneca-NGS/VarDictJava" + documentation: "https://github.com/AstraZeneca-NGS/VarDictJava" + tool_dev_url: "https://github.com/AstraZeneca-NGS/VarDictJava" + doi: "10.1093/nar/gkw227 " licence: "['MIT']" -## TODO nf-core: Add a description of all of the variables used as input input: # Only when we have meta - meta: @@ -21,14 +21,22 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - # - ## TODO nf-core: Delete / customise this example input + - bam: type: file - description: BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + description: BAM/SAM file + pattern: "*.{bam,sam}" + + - reference_fasta: + type: file + description: FASTA of the reference genome + pattern: "*.{fa,fasta}" + + - regions_of_interest: + type: file + description: BED with the regions of interest + pattern: "*.bed" -## TODO nf-core: Add a description of all of the variables used as output output: #Only when we have meta - meta: @@ -36,16 +44,16 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - # + - versions: type: file description: File containing software versions pattern: "versions.yml" - ## TODO nf-core: Delete / customise this example output - - bam: + + - vcf: type: file - description: Sorted BAM/CRAM/SAM file - pattern: "*.{bam,cram,sam}" + description: VCF file output + pattern: "*.vcf" authors: - "@nvnieuwk" diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index 8a714b2b..7511b529 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -8,8 +8,12 @@ workflow test_vardictjava { input = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_bam'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) ] - VARDICTJAVA ( input ) + VARDICTJAVA ( + input, + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + ) } diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 73c647af..001a56e8 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -6,7 +6,7 @@ - "vardictjava" # files: - - path: "output/vardictjava/test.bam" + - path: "output/vardictjava/test.vcf" md5sum: e667c7caad0bc4b7ac383fd023c654fc - path: output/vardictjava/versions.yml md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From 839ee59ca1646d055dced78125d88d8a864cf651 Mon Sep 17 00:00:00 2001 From: jvhagey Date: Wed, 27 Apr 2022 18:01:28 -0400 Subject: [PATCH 19/91] adding srst2 module --- modules/srst2/srst2/main.nf | 47 ++++++++++++++++ modules/srst2/srst2/meta.yml | 68 +++++++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/srst2/srst2/main.nf | 28 ++++++++++ tests/modules/srst2/srst2/nextflow.config | 5 ++ tests/modules/srst2/srst2/test.yml | 29 ++++++++++ 6 files changed, 181 insertions(+) create mode 100644 modules/srst2/srst2/main.nf create mode 100644 modules/srst2/srst2/meta.yml create mode 100644 tests/modules/srst2/srst2/main.nf create mode 100644 tests/modules/srst2/srst2/nextflow.config create mode 100644 tests/modules/srst2/srst2/test.yml diff --git a/modules/srst2/srst2/main.nf b/modules/srst2/srst2/main.nf new file mode 100644 index 00000000..fc665fad --- /dev/null +++ b/modules/srst2/srst2/main.nf @@ -0,0 +1,47 @@ +process SRST2_SRST2 { + tag "${meta.id}" + label 'process_low' + + conda (params.enable_conda ? "bioconda::srst2=0.2.0" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/srst2%3A0.2.0--py_4': + 'quay.io/biocontainers/srst2:0.2.0--py_4'}" + + input: + tuple val(meta), path(fastq_s), path(db) + + output: + tuple val(meta), path("*_genes_*_results.txt") , emit: gene_results + tuple val(meta), path("*_fullgenes_*_results.txt") , optional:true, emit: fullgene_results + tuple val(meta), path("*.pileup") , emit: pileup + tuple val(meta), path("*.sorted.bam") , emit: sorted_bam + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: "" + def prefix = task.ext.prefix ?: "${meta.id}" + def read_s = meta.single_end ? "--input_se ${fastq_s}" : "--input_pe ${fastq_s[0]} ${fastq_s[1]}" + if (meta.db=="gene") { + database = "--gene_db ${db}" + } else if (meta.db=="mlst") { + database = "--mlst_db ${db}" + } else { + return + } + """ + srst2 \\ + ${read_s} \\ + --threads $task.cpus \\ + --output ${prefix} \\ + ${database} \\ + $args + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + srst2: \$(echo \$(srst2 --version 2>&1) | sed 's/srst2 //' )) + END_VERSIONS + """ +} diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml new file mode 100644 index 00000000..2c44f53d --- /dev/null +++ b/modules/srst2/srst2/meta.yml @@ -0,0 +1,68 @@ +name: srst2_srst2 +description: | + Short Read Sequence Typing for Bacterial Pathogens is a program designed to take Illumina sequence data, + a MLST database and/or a database of gene sequences (e.g. resistance genes, virulence genes, etc) + and report the presence of STs and/or reference genes. +keywords: + - mlst + - typing + - illumina +tools: + - srst2: + description: "Short Read Sequence Typing for Bacterial Pathogens" + homepage: {http://katholt.github.io/srst2/} + documentation: {https://github.com/katholt/srst2/blob/master/README.md} + tool_dev_url: {https://github.com/katholt/srst2} + doi: "" + licence: ['BSD'] + +input: + - meta: + type: map0.2.0-4 + description: | + Groovy Map containing sample information + id: should be the identification number or sample name + single_end: should be true for single end data and false for paired in data + db: should be either 'gene' to use the --gene_db option or "mlst" to use the --mlst_db option + e.g. [ id:'sample', single_end:false , db:'gene'] + - fasta: + type: file + description: | + gzipped fasta file. If files are NOT in + MiSeq format sample_S1_L001_R1_001.fastq.gz uses --forward and --reverse parameters; otherwise + default is _1, i.e. expect forward reads as sample_1.fastq.gz). + pattern: "*.fastq.gz" + - db: + type: file + description: Database in FASTA format + pattern: "*.fasta" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'sample', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - txt: + type: file + description: results text file + pattern: "*_fullgenes_*_results.txt" + - txt: + type: file + description: results text file + pattern: "*_genes_*_results.txt" + - bam: + type: file + description: Sorted BAM file + pattern: "*.sorted.bam" + - pileup: + type: file + description: SAMtools pileup file + pattern: "*.pileup" + +authors: + - "@jvhagey" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 4d8ce0b5..66f50903 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1767,6 +1767,10 @@ sratools/prefetch: - modules/sratools/prefetch/** - tests/modules/sratools/prefetch/** +srst2/srst2: + - modules/srst2/srst2/** + - tests/modules/srst2/srst2/** + ssuissero: - modules/ssuissero/** - tests/modules/ssuissero/** diff --git a/tests/modules/srst2/srst2/main.nf b/tests/modules/srst2/srst2/main.nf new file mode 100644 index 00000000..235f3ff9 --- /dev/null +++ b/tests/modules/srst2/srst2/main.nf @@ -0,0 +1,28 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { SRST2_SRST2 } from '../../../../modules/srst2/srst2/main.nf' + +workflow test_srst2_srst2_paired_end { + + input = [ + [ id:'test', single_end:false, db:"gene"], // meta map + [ file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true), + file(params.test_data['bacteroides_fragilis']['illumina']['test1_2_fastq_gz'], checkIfExists: true) ], + file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/resFinder_20180221_srst2.fasta') // Change to params.test_data syntax after the data is included in tests/config/test_data.config + ] + + SRST2_SRST2(input) +} + +workflow test_srst2_srst2_single_end { + + input = [ + [ id:'test', single_end:true, db:"gene" ], // meta map + file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true), + file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/resFinder_20180221_srst2.fasta') // Change to params.test_data syntax after the data is included in tests/config/test_data.config + ] + + SRST2_SRST2(input) +} \ No newline at end of file diff --git a/tests/modules/srst2/srst2/nextflow.config b/tests/modules/srst2/srst2/nextflow.config new file mode 100644 index 00000000..50f50a7a --- /dev/null +++ b/tests/modules/srst2/srst2/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml new file mode 100644 index 00000000..7afd0687 --- /dev/null +++ b/tests/modules/srst2/srst2/test.yml @@ -0,0 +1,29 @@ +- name: srst2 srst2 test_srst2_srst2_paired_end + command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_paired_end -c tests/config/nextflow.config + tags: + - srst2/srst2 + - srst2 + files: + - path: /tmp/tmp03v2z3fu/srst2/test__genes__resFinder_20180221_srst2__results.txt + md5sum: 099aa6cacec5524b311f606debdfb3a9 + - path: /tmp/tmp03v2z3fu/srst2/test__test1.resFinder_20180221_srst2.pileup + md5sum: 64b512ff495b828c456405ec7b676ad1 + - path: /tmp/tmp03v2z3fu/srst2/test__test1.resFinder_20180221_srst2.sorted.bam + - path: /tmp/tmp03v2z3fu/srst2/versions.yml + md5sum: b446a70f1a2b4f60757829bcd744a214 + +- name: srst2 srst2 test_srst2_srst2_single_end + command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_single_end -c tests/config/nextflow.config + tags: + - srst2/srst2 + - srst2 + files: + - path: /tmp/tmp3kabyta4/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt + md5sum: d0762ef8c38afd0e0a34cce52ed1a3db + - path: /tmp/tmp3kabyta4/srst2/test__genes__resFinder_20180221_srst2__results.txt + md5sum: b8850c6644406d8b131e471ecc3f9013 + - path: /tmp/tmp3kabyta4/srst2/test__test1_1.resFinder_20180221_srst2.pileup + md5sum: 5f6279dc8124aa762a9dfe3d7a871277 + - path: /tmp/tmp3kabyta4/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam + - path: /tmp/tmp3kabyta4/srst2/versions.yml + md5sum: 790fe00493c6634d17801a930073218b From a575b91254480f964aa4e105f8fab09d0ecd8451 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Wed, 27 Apr 2022 19:54:33 -0400 Subject: [PATCH 20/91] Update test.yml --- tests/modules/srst2/srst2/test.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 7afd0687..f7d058de 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -1,29 +1,29 @@ - name: srst2 srst2 test_srst2_srst2_paired_end command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_paired_end -c tests/config/nextflow.config tags: - - srst2/srst2 - srst2 + - srst2/srst2 files: - - path: /tmp/tmp03v2z3fu/srst2/test__genes__resFinder_20180221_srst2__results.txt + - path: /tmp/tmpsqtj3g4l/srst2/test__genes__resFinder_20180221_srst2__results.txt md5sum: 099aa6cacec5524b311f606debdfb3a9 - - path: /tmp/tmp03v2z3fu/srst2/test__test1.resFinder_20180221_srst2.pileup + - path: /tmp/tmpsqtj3g4l/srst2/test__test1.resFinder_20180221_srst2.pileup md5sum: 64b512ff495b828c456405ec7b676ad1 - - path: /tmp/tmp03v2z3fu/srst2/test__test1.resFinder_20180221_srst2.sorted.bam - - path: /tmp/tmp03v2z3fu/srst2/versions.yml + - path: /tmp/tmpsqtj3g4l/srst2/test__test1.resFinder_20180221_srst2.sorted.bam + - path: /tmp/tmpsqtj3g4l/srst2/versions.yml md5sum: b446a70f1a2b4f60757829bcd744a214 - name: srst2 srst2 test_srst2_srst2_single_end command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_single_end -c tests/config/nextflow.config tags: - - srst2/srst2 - srst2 + - srst2/srst2 files: - - path: /tmp/tmp3kabyta4/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt + - path: /tmp/tmpo9au1v02/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt md5sum: d0762ef8c38afd0e0a34cce52ed1a3db - - path: /tmp/tmp3kabyta4/srst2/test__genes__resFinder_20180221_srst2__results.txt + - path: /tmp/tmpo9au1v02/srst2/test__genes__resFinder_20180221_srst2__results.txt md5sum: b8850c6644406d8b131e471ecc3f9013 - - path: /tmp/tmp3kabyta4/srst2/test__test1_1.resFinder_20180221_srst2.pileup + - path: /tmp/tmpo9au1v02/srst2/test__test1_1.resFinder_20180221_srst2.pileup md5sum: 5f6279dc8124aa762a9dfe3d7a871277 - - path: /tmp/tmp3kabyta4/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam - - path: /tmp/tmp3kabyta4/srst2/versions.yml + - path: /tmp/tmpo9au1v02/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam + - path: /tmp/tmpo9au1v02/srst2/versions.yml md5sum: 790fe00493c6634d17801a930073218b From ecd31fd3cdc1646bb80e5b04a52cfc3e4666b18b Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Wed, 27 Apr 2022 19:55:42 -0400 Subject: [PATCH 21/91] remove whitespaces and prettier fix --- modules/srst2/srst2/meta.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml index 2c44f53d..31cf4de0 100644 --- a/modules/srst2/srst2/meta.yml +++ b/modules/srst2/srst2/meta.yml @@ -1,7 +1,7 @@ name: srst2_srst2 description: | - Short Read Sequence Typing for Bacterial Pathogens is a program designed to take Illumina sequence data, - a MLST database and/or a database of gene sequences (e.g. resistance genes, virulence genes, etc) + Short Read Sequence Typing for Bacterial Pathogens is a program designed to take Illumina sequence data, + a MLST database and/or a database of gene sequences (e.g. resistance genes, virulence genes, etc) and report the presence of STs and/or reference genes. keywords: - mlst @@ -10,11 +10,11 @@ keywords: tools: - srst2: description: "Short Read Sequence Typing for Bacterial Pathogens" - homepage: {http://katholt.github.io/srst2/} - documentation: {https://github.com/katholt/srst2/blob/master/README.md} - tool_dev_url: {https://github.com/katholt/srst2} + homepage: { http://katholt.github.io/srst2/ } + documentation: { https://github.com/katholt/srst2/blob/master/README.md } + tool_dev_url: { https://github.com/katholt/srst2 } doi: "" - licence: ['BSD'] + licence: ["BSD"] input: - meta: @@ -49,11 +49,11 @@ output: pattern: "versions.yml" - txt: type: file - description: results text file + description: results text file pattern: "*_fullgenes_*_results.txt" - txt: type: file - description: results text file + description: results text file pattern: "*_genes_*_results.txt" - bam: type: file From b0c5f9422bf4f48a6799b42e9302fe71c0e9bf0f Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Wed, 27 Apr 2022 20:21:41 -0400 Subject: [PATCH 22/91] tmp/ to changed to output/ --- tests/modules/srst2/srst2/test.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index f7d058de..f011b4d1 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -4,12 +4,12 @@ - srst2 - srst2/srst2 files: - - path: /tmp/tmpsqtj3g4l/srst2/test__genes__resFinder_20180221_srst2__results.txt + - path: output/srst2/test__genes__resFinder_20180221_srst2__results.txt md5sum: 099aa6cacec5524b311f606debdfb3a9 - - path: /tmp/tmpsqtj3g4l/srst2/test__test1.resFinder_20180221_srst2.pileup + - path: output/srst2/test__test1.resFinder_20180221_srst2.pileup md5sum: 64b512ff495b828c456405ec7b676ad1 - - path: /tmp/tmpsqtj3g4l/srst2/test__test1.resFinder_20180221_srst2.sorted.bam - - path: /tmp/tmpsqtj3g4l/srst2/versions.yml + - path: output/srst2/test__test1.resFinder_20180221_srst2.sorted.bam + - path: output/srst2/versions.yml md5sum: b446a70f1a2b4f60757829bcd744a214 - name: srst2 srst2 test_srst2_srst2_single_end @@ -18,12 +18,12 @@ - srst2 - srst2/srst2 files: - - path: /tmp/tmpo9au1v02/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt + - path: output/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt md5sum: d0762ef8c38afd0e0a34cce52ed1a3db - - path: /tmp/tmpo9au1v02/srst2/test__genes__resFinder_20180221_srst2__results.txt + - path: output/srst2/test__genes__resFinder_20180221_srst2__results.txt md5sum: b8850c6644406d8b131e471ecc3f9013 - - path: /tmp/tmpo9au1v02/srst2/test__test1_1.resFinder_20180221_srst2.pileup + - path: output/srst2/test__test1_1.resFinder_20180221_srst2.pileup md5sum: 5f6279dc8124aa762a9dfe3d7a871277 - - path: /tmp/tmpo9au1v02/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam - - path: /tmp/tmpo9au1v02/srst2/versions.yml + - path: output/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam + - path: output/srst2/versions.yml md5sum: 790fe00493c6634d17801a930073218b From 85c49a971f91208ead572212be8475f5e7518feb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 09:07:52 +0200 Subject: [PATCH 23/91] Fixed an issue with wrong file order --- modules/vardictjava/main.nf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 8170bf76..855784db 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -8,9 +8,8 @@ process VARDICTJAVA { 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - tuple val(meta), path(bam) + tuple val(meta), path(bam), path(regions_of_interest) path(reference_fasta) - path(regions_of_interest) output: tuple val(meta), path("*.vcf"), emit: vcf @@ -24,6 +23,9 @@ process VARDICTJAVA { def prefix = task.ext.prefix ?: "${meta.id}" """ + head -n 20 $reference_fasta + cat $reference_fasta | wc -l + vardict-java \\ $args \\ -b $bam \\ @@ -31,10 +33,11 @@ process VARDICTJAVA { -n $prefix \\ -G $reference_fasta \\ $regions_of_interest \\ + > ${prefix}.vcf cat <<-END_VERSIONS > versions.yml "${task.process}": - vardictjava: \$(echo 1.8.3) + vardict-java: \$(echo 1.8.3) END_VERSIONS """ } From db43095b1a1530af1e3cac7f3e7211c52d87bbd7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 09:18:15 +0200 Subject: [PATCH 24/91] Split the bed file into a seperate input --- modules/vardictjava/main.nf | 8 +++----- tests/modules/vardictjava/main.nf | 14 +++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 855784db..e1098b22 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -8,7 +8,8 @@ process VARDICTJAVA { 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - tuple val(meta), path(bam), path(regions_of_interest) + tuple val(meta), path(bam) + path(regions_of_interest) path(reference_fasta) output: @@ -23,14 +24,11 @@ process VARDICTJAVA { def prefix = task.ext.prefix ?: "${meta.id}" """ - head -n 20 $reference_fasta - cat $reference_fasta | wc -l - vardict-java \\ $args \\ -b $bam \\ -th $task.cpus \\ - -n $prefix \\ + -N $prefix \\ -G $reference_fasta \\ $regions_of_interest \\ > ${prefix}.vcf diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index 7511b529..647c8bba 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -6,14 +6,14 @@ include { VARDICTJAVA } from '../../../modules/vardictjava/main.nf' workflow test_vardictjava { - input = [ + bam_input_ch = Channel.of([ [ id:'test', single_end:false ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) - ] + ]) - VARDICTJAVA ( - input, - file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), - file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) - ) + bed = file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + + fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) + + VARDICTJAVA ( bam_input_ch, bed, fasta ) } From def580ddb7af11619f8f0757ef5af5ed156ed6a4 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 28 Apr 2022 09:28:57 +0200 Subject: [PATCH 25/91] Add gzip for compressed output --- modules/vardictjava/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index e1098b22..425c1706 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -31,7 +31,7 @@ process VARDICTJAVA { -N $prefix \\ -G $reference_fasta \\ $regions_of_interest \\ - > ${prefix}.vcf + | gzip -c > ${prefix}.vcf.gz cat <<-END_VERSIONS > versions.yml "${task.process}": From 484c8f5dfa40a2117997afcd8434bcd52ff4fe4e Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 28 Apr 2022 09:29:27 +0200 Subject: [PATCH 26/91] Update meta.yml --- modules/vardictjava/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 7820a877..5bc21b55 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -53,7 +53,7 @@ output: - vcf: type: file description: VCF file output - pattern: "*.vcf" + pattern: "*.vcf.gz" authors: - "@nvnieuwk" From 9df830f79dc70a347578120e797a4e3160dfc66d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 10:23:07 +0200 Subject: [PATCH 27/91] Added the .bai and .fai files --- modules/vardictjava/main.nf | 8 ++++---- tests/modules/vardictjava/main.nf | 12 ++++++++---- tests/modules/vardictjava/test.yml | 15 ++++++--------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 425c1706..c370b6ef 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -8,12 +8,12 @@ process VARDICTJAVA { 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - tuple val(meta), path(bam) + tuple val(meta), path(bam), path(bai) path(regions_of_interest) - path(reference_fasta) + tuple path(reference_fasta), path(reference_fai) output: - tuple val(meta), path("*.vcf"), emit: vcf + tuple val(meta), path("*.vcf.gz"), emit: vcf path "versions.yml" , emit: versions when: @@ -30,7 +30,7 @@ process VARDICTJAVA { -th $task.cpus \\ -N $prefix \\ -G $reference_fasta \\ - $regions_of_interest \\ + $regions_of_interest | gzip -c > ${prefix}.vcf.gz cat <<-END_VERSIONS > versions.yml diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index 647c8bba..89eec407 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -8,12 +8,16 @@ workflow test_vardictjava { bam_input_ch = Channel.of([ [ id:'test', single_end:false ], // meta map - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) ]) - bed = file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + bed = file(params.test_data['homo_sapiens']['genome']['genome_bed_gz'], checkIfExists: true) - fasta = file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true) + reference = Channel.of([ + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ]) - VARDICTJAVA ( bam_input_ch, bed, fasta ) + VARDICTJAVA ( bam_input_ch, bed, reference ) } diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 001a56e8..77aa9047 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -1,12 +1,9 @@ -## TODO nf-core: Please run the following command to build this file: -# nf-core modules create-test-yml vardictjava -- name: "vardictjava" - command: nextflow run ./tests/modules/vardictjava -entry test_vardictjava -c ./tests/config/nextflow.config -c ./tests/modules/vardictjava/nextflow.config +- name: vardictjava test_vardictjava + command: nextflow run tests/modules/vardictjava -entry test_vardictjava -c tests/config/nextflow.config tags: - - "vardictjava" - # + - vardictjava files: - - path: "output/vardictjava/test.vcf" - md5sum: e667c7caad0bc4b7ac383fd023c654fc + - path: output/vardictjava/test.vcf.gz + md5sum: 2179dcaee6183495b421293f42db11b5 - path: output/vardictjava/versions.yml - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b + md5sum: aac455b8c9c9194c5fed80e4fd495b96 From 45b4a61a15c8b7ac6c29a447ccc1ece9f08f8efe Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 13:31:37 +0200 Subject: [PATCH 28/91] Some small updates --- modules/vardictjava/main.nf | 2 +- tests/modules/vardictjava/main.nf | 8 ++++---- tests/modules/vardictjava/test.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index c370b6ef..26833436 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -30,7 +30,7 @@ process VARDICTJAVA { -th $task.cpus \\ -N $prefix \\ -G $reference_fasta \\ - $regions_of_interest + $regions_of_interest \\ | gzip -c > ${prefix}.vcf.gz cat <<-END_VERSIONS > versions.yml diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index 89eec407..d392358a 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -6,15 +6,15 @@ include { VARDICTJAVA } from '../../../modules/vardictjava/main.nf' workflow test_vardictjava { - bam_input_ch = Channel.of([ - [ id:'test', single_end:false ], // meta map + bam_input_ch = Channel.value([ + [ id:'test' ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) ]) - bed = file(params.test_data['homo_sapiens']['genome']['genome_bed_gz'], checkIfExists: true) + bed = Channel.value(file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true)) - reference = Channel.of([ + reference = Channel.value([ file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) ]) diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 77aa9047..0bd21f4e 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -4,6 +4,6 @@ - vardictjava files: - path: output/vardictjava/test.vcf.gz - md5sum: 2179dcaee6183495b421293f42db11b5 + md5sum: 7029066c27ac6f5ef18d660d5741979a - path: output/vardictjava/versions.yml md5sum: aac455b8c9c9194c5fed80e4fd495b96 From 8c98a78d2a0e12d081184736e17ccde3feee3f44 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 14:07:55 +0200 Subject: [PATCH 29/91] Fixed the issue with the BED file --- modules/vardictjava/main.nf | 1 + tests/modules/vardictjava/test.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 26833436..d6b4f1dd 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -26,6 +26,7 @@ process VARDICTJAVA { """ vardict-java \\ $args \\ + -c 1 -S 2 -E 3 \\ -b $bam \\ -th $task.cpus \\ -N $prefix \\ diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 0bd21f4e..77aa9047 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -4,6 +4,6 @@ - vardictjava files: - path: output/vardictjava/test.vcf.gz - md5sum: 7029066c27ac6f5ef18d660d5741979a + md5sum: 2179dcaee6183495b421293f42db11b5 - path: output/vardictjava/versions.yml md5sum: aac455b8c9c9194c5fed80e4fd495b96 From bf186044716cb0621f31ccc4727ad4a0fb7c81b9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 14:29:37 +0200 Subject: [PATCH 30/91] fix for a linting issue --- modules/vardictjava/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index d6b4f1dd..3e036ebd 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -14,7 +14,7 @@ process VARDICTJAVA { output: tuple val(meta), path("*.vcf.gz"), emit: vcf - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions when: task.ext.when == null || task.ext.when From 40e6e5273de785042fc1b4d6fc1d7510729a7789 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 14:30:04 +0200 Subject: [PATCH 31/91] fix for a linting issue --- modules/vardictjava/meta.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 5bc21b55..39368f81 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -15,7 +15,6 @@ tools: licence: "['MIT']" input: - # Only when we have meta - meta: type: map description: | @@ -38,7 +37,6 @@ input: pattern: "*.bed" output: - #Only when we have meta - meta: type: map description: | From 6678ad4426664076d912dab488ea641f44c1fdb6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 14:54:20 +0200 Subject: [PATCH 32/91] Added a header to the VCF --- modules/vardictjava/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 3e036ebd..a28cb539 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -26,7 +26,7 @@ process VARDICTJAVA { """ vardict-java \\ $args \\ - -c 1 -S 2 -E 3 \\ + -c 1 -S 2 -E 3 -h \\ -b $bam \\ -th $task.cpus \\ -N $prefix \\ From bdff37c3d7c880f599af223504808306e75e2d76 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 15:33:27 +0200 Subject: [PATCH 33/91] Added the statistical tests and conversion to vcf --- modules/vardictjava/main.nf | 7 ++++++- tests/modules/vardictjava/nextflow.config | 1 + tests/modules/vardictjava/test.yml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index a28cb539..92051074 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -21,17 +21,22 @@ process VARDICTJAVA { script: def args = task.ext.args ?: '' + def args_conversion = task.ext.args_conversion ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ vardict-java \\ $args \\ - -c 1 -S 2 -E 3 -h \\ + -c 1 -S 2 -E 3 \\ -b $bam \\ -th $task.cpus \\ -N $prefix \\ -G $reference_fasta \\ $regions_of_interest \\ + | teststrandbias.R \\ + | var2vcf_valid.pl \\ + $args_conversion \\ + -N $prefix \\ | gzip -c > ${prefix}.vcf.gz cat <<-END_VERSIONS > versions.yml diff --git a/tests/modules/vardictjava/nextflow.config b/tests/modules/vardictjava/nextflow.config index 5dc176a9..fc698f59 100644 --- a/tests/modules/vardictjava/nextflow.config +++ b/tests/modules/vardictjava/nextflow.config @@ -4,5 +4,6 @@ process { withName: VARDICTJAVA { ext.args = '' + ext.args_conversion = '' } } \ No newline at end of file diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 77aa9047..549d688e 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -4,6 +4,6 @@ - vardictjava files: - path: output/vardictjava/test.vcf.gz - md5sum: 2179dcaee6183495b421293f42db11b5 + md5sum: 3f1f227afc532bddeb58f16fd3013fc8 - path: output/vardictjava/versions.yml md5sum: aac455b8c9c9194c5fed80e4fd495b96 From d626a1dcd3ca759c2c398efe49642a39d843ecd7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 15:41:43 +0200 Subject: [PATCH 34/91] Removed a whitespace in meta.yml... --- modules/vardictjava/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 39368f81..e3b2efe7 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -25,7 +25,7 @@ input: type: file description: BAM/SAM file pattern: "*.{bam,sam}" - + - reference_fasta: type: file description: FASTA of the reference genome From 4b04ca22ddc577951bc59a693e247c6ff5949dd7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Thu, 28 Apr 2022 16:38:49 +0200 Subject: [PATCH 35/91] Variable name change for var2vcf conversion arguments --- modules/vardictjava/main.nf | 4 ++-- tests/modules/vardictjava/nextflow.config | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 92051074..08318c29 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -21,7 +21,7 @@ process VARDICTJAVA { script: def args = task.ext.args ?: '' - def args_conversion = task.ext.args_conversion ?: '' + def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ @@ -35,7 +35,7 @@ process VARDICTJAVA { $regions_of_interest \\ | teststrandbias.R \\ | var2vcf_valid.pl \\ - $args_conversion \\ + $args2 \\ -N $prefix \\ | gzip -c > ${prefix}.vcf.gz diff --git a/tests/modules/vardictjava/nextflow.config b/tests/modules/vardictjava/nextflow.config index fc698f59..e08201cc 100644 --- a/tests/modules/vardictjava/nextflow.config +++ b/tests/modules/vardictjava/nextflow.config @@ -4,6 +4,6 @@ process { withName: VARDICTJAVA { ext.args = '' - ext.args_conversion = '' + ext.args2 = '' } } \ No newline at end of file From c7408538746bd2c8d64418691c0e0440483c48da Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:42:44 +0200 Subject: [PATCH 36/91] Antismash db output patch 1 (#3) * Update main.nf * Update meta.yml * Update test.yml --- .../antismashlitedownloaddatabases/main.nf | 6 +++--- .../antismashlitedownloaddatabases/meta.yml | 18 ++++-------------- .../antismashlitedownloaddatabases/test.yml | 9 ++++----- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index 72314eee..5f9141f0 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -27,9 +27,7 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { output: path("antismash_db") , emit: database - path("css"), emit: css_dir - path("detection"), emit: detection_dir - path("modules"), emit: modules_dir + path("antismash_dir"), emit: antismash_dir path "versions.yml", emit: versions when: @@ -42,6 +40,8 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { --database-dir antismash_db \\ $args + cp -r /usr/local/lib/python3.8/site-packages/antismash antismash_dir + cat <<-END_VERSIONS > versions.yml "${task.process}": antismash-lite: \$(antismash --version | sed 's/antiSMASH //') diff --git a/modules/antismash/antismashlitedownloaddatabases/meta.yml b/modules/antismash/antismashlitedownloaddatabases/meta.yml index 619dc8c2..4b9644c4 100644 --- a/modules/antismash/antismashlitedownloaddatabases/meta.yml +++ b/modules/antismash/antismashlitedownloaddatabases/meta.yml @@ -27,7 +27,7 @@ input: - database_css: type: directory description: | - antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by ther use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "css" - database_detection: type: directory @@ -50,21 +50,11 @@ output: type: directory description: Download directory for antiSMASH databases pattern: "antismash_db" - - css_dir: + - antismash_dir: type: directory description: | - antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. - pattern: "css" - - detection_dir: - type: directory - description: | - antismash/detection folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. - pattern: "detection" - - modules_dir: - type: directory - description: | - antismash/modules folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. - pattern: "modules" + antismash installation folder which is being modified during the antiSMASH database downloading step. The modified files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database and installation folder in pipelines. + pattern: "antismash_dir" authors: - "@jasmezz" diff --git a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml index 1079363c..f9143163 100644 --- a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml +++ b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml @@ -1,17 +1,16 @@ - name: antismash antismashlitedownloaddatabases test_antismash_antismashlitedownloaddatabases command: nextflow run tests/modules/antismash/antismashlitedownloaddatabases -entry test_antismash_antismashlitedownloaddatabases -c tests/config/nextflow.config tags: - - antismash - antismash/antismashlitedownloaddatabases + - antismash files: - path: output/antismash/versions.yml - md5sum: 24859c67023abab99de295d3675a24b6 + md5sum: e2656c8d2bcc7469eba40eb1ee5c91b3 - path: output/antismash/antismash_db - path: output/antismash/antismash_db/clusterblast - path: output/antismash/antismash_db/clustercompare - path: output/antismash/antismash_db/pfam - path: output/antismash/antismash_db/resfam - path: output/antismash/antismash_db/tigrfam - - path: output/antismash/css - - path: output/antismash/detection - - path: output/antismash/modules + - path: output/antismash/antismash_dir + - path: output/antismash/antismash_dir/detection/hmm_detection/data/bgc_seeds.hmm From 39258e53d7b6168fc576d1d46dbd78c286919ec1 Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Thu, 28 Apr 2022 17:09:40 +0200 Subject: [PATCH 37/91] Update checksum in test.yml --- tests/modules/antismash/antismashlitedownloaddatabases/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml index f9143163..b4a964a0 100644 --- a/tests/modules/antismash/antismashlitedownloaddatabases/test.yml +++ b/tests/modules/antismash/antismashlitedownloaddatabases/test.yml @@ -5,7 +5,7 @@ - antismash files: - path: output/antismash/versions.yml - md5sum: e2656c8d2bcc7469eba40eb1ee5c91b3 + md5sum: 24859c67023abab99de295d3675a24b6 - path: output/antismash/antismash_db - path: output/antismash/antismash_db/clusterblast - path: output/antismash/antismash_db/clustercompare From 5be6afd6b1e9b4362ad9f3e43e4a93e0ef4af423 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Thu, 28 Apr 2022 12:29:39 -0400 Subject: [PATCH 38/91] changed container --- modules/srst2/srst2/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/srst2/srst2/main.nf b/modules/srst2/srst2/main.nf index fc665fad..5f69205d 100644 --- a/modules/srst2/srst2/main.nf +++ b/modules/srst2/srst2/main.nf @@ -4,8 +4,8 @@ process SRST2_SRST2 { conda (params.enable_conda ? "bioconda::srst2=0.2.0" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/srst2%3A0.2.0--py_4': - 'quay.io/biocontainers/srst2:0.2.0--py_4'}" + 'https://depot.galaxyproject.org/singularity/srst2%3A0.2.0--py27_2': + 'quay.io/biocontainers/srst2:0.2.0--py27_2'}" input: tuple val(meta), path(fastq_s), path(db) From e29a69077d470734cd721be8932a1e5b6bf64ff4 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Thu, 28 Apr 2022 12:30:04 -0400 Subject: [PATCH 39/91] add doi --- modules/srst2/srst2/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml index 31cf4de0..17d9226d 100644 --- a/modules/srst2/srst2/meta.yml +++ b/modules/srst2/srst2/meta.yml @@ -13,7 +13,7 @@ tools: homepage: { http://katholt.github.io/srst2/ } documentation: { https://github.com/katholt/srst2/blob/master/README.md } tool_dev_url: { https://github.com/katholt/srst2 } - doi: "" + doi: "10.1186/s13073-014-0090-6" licence: ["BSD"] input: From 9e7e10305b98e57f221de07d56bc7add2cfbd263 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Thu, 28 Apr 2022 14:02:28 -0400 Subject: [PATCH 40/91] Update modules/srst2/srst2/meta.yml Co-authored-by: Sateesh Peri <33637490+sateeshperi@users.noreply.github.com> --- modules/srst2/srst2/meta.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml index 17d9226d..f70d3780 100644 --- a/modules/srst2/srst2/meta.yml +++ b/modules/srst2/srst2/meta.yml @@ -10,9 +10,9 @@ keywords: tools: - srst2: description: "Short Read Sequence Typing for Bacterial Pathogens" - homepage: { http://katholt.github.io/srst2/ } - documentation: { https://github.com/katholt/srst2/blob/master/README.md } - tool_dev_url: { https://github.com/katholt/srst2 } + homepage: "http://katholt.github.io/srst2/" + documentation: "https://github.com/katholt/srst2/blob/master/README.md" + tool_dev_url: "https://github.com/katholt/srst2" doi: "10.1186/s13073-014-0090-6" licence: ["BSD"] From 842ae3982bd2238cbf5a6e50874c287962573cd9 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Thu, 28 Apr 2022 14:39:02 -0400 Subject: [PATCH 41/91] Update meta.yml --- modules/srst2/srst2/meta.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml index f70d3780..3755fb34 100644 --- a/modules/srst2/srst2/meta.yml +++ b/modules/srst2/srst2/meta.yml @@ -49,11 +49,11 @@ output: pattern: "versions.yml" - txt: type: file - description: results text file + description: A detailed report, with one row per gene per sample described here: https://github.com/katholt/srst2#gene-typing pattern: "*_fullgenes_*_results.txt" - txt: type: file - description: results text file + description: A tabulated summary report of samples x genes. pattern: "*_genes_*_results.txt" - bam: type: file From 218769d7df89c78ed253f4a5a70b436e4de17eba Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Thu, 28 Apr 2022 14:41:37 -0400 Subject: [PATCH 42/91] Update modules/srst2/srst2/main.nf Co-authored-by: Edmund Miller --- modules/srst2/srst2/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/srst2/srst2/main.nf b/modules/srst2/srst2/main.nf index 5f69205d..4ee9f6a6 100644 --- a/modules/srst2/srst2/main.nf +++ b/modules/srst2/srst2/main.nf @@ -29,7 +29,7 @@ process SRST2_SRST2 { } else if (meta.db=="mlst") { database = "--mlst_db ${db}" } else { - return + error "Please set meta.db to either \"gene\" or \"mlst\"" } """ srst2 \\ From 6b64f9cb6c3dd3577931cc3cd032d6fb730000ce Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 10:34:14 +0200 Subject: [PATCH 43/91] feat added stub to modules --- modules/gatk4/mergebamalignment/main.nf | 11 +++++++++++ modules/gatk4/mutect2/main.nf | 14 ++++++++++++++ modules/gatk4/revertsam/main.nf | 11 +++++++++++ modules/gatk4/samtofastq/main.nf | 13 +++++++++++++ modules/samtools/view/main.nf | 12 ++++++++++++ 5 files changed, 61 insertions(+) diff --git a/modules/gatk4/mergebamalignment/main.nf b/modules/gatk4/mergebamalignment/main.nf index 7ba9ccda..5c36b4ba 100644 --- a/modules/gatk4/mergebamalignment/main.nf +++ b/modules/gatk4/mergebamalignment/main.nf @@ -43,4 +43,15 @@ process GATK4_MERGEBAMALIGNMENT { gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ } diff --git a/modules/gatk4/mutect2/main.nf b/modules/gatk4/mutect2/main.nf index 4a1f5768..a214b57d 100644 --- a/modules/gatk4/mutect2/main.nf +++ b/modules/gatk4/mutect2/main.nf @@ -57,4 +57,18 @@ process GATK4_MUTECT2 { gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.vcf.gz + touch ${prefix}.tbi + touch ${prefix}.stats + touch ${prefix}.f1r2.tar.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ } diff --git a/modules/gatk4/revertsam/main.nf b/modules/gatk4/revertsam/main.nf index 4e8e9ddc..3084658d 100644 --- a/modules/gatk4/revertsam/main.nf +++ b/modules/gatk4/revertsam/main.nf @@ -39,4 +39,15 @@ process GATK4_REVERTSAM { gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.reverted.bam + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ } diff --git a/modules/gatk4/samtofastq/main.nf b/modules/gatk4/samtofastq/main.nf index 8553e419..d8d94d69 100644 --- a/modules/gatk4/samtofastq/main.nf +++ b/modules/gatk4/samtofastq/main.nf @@ -40,4 +40,17 @@ process GATK4_SAMTOFASTQ { gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.fastq.gz + touch ${prefix}_1.fastq.gz + touch ${prefix}_2.fastq.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//') + END_VERSIONS + """ } diff --git a/modules/samtools/view/main.nf b/modules/samtools/view/main.nf index 11cfb74b..55194e88 100644 --- a/modules/samtools/view/main.nf +++ b/modules/samtools/view/main.nf @@ -41,4 +41,16 @@ process SAMTOOLS_VIEW { samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.bam + touch ${prefix}.cram + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//') + END_VERSIONS + """ } From 8a5bcbc325d20f081c9a722bc47f380df43448b7 Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 11:06:08 +0200 Subject: [PATCH 44/91] fix the stub of gatk4_revertsam --- modules/gatk4/revertsam/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gatk4/revertsam/main.nf b/modules/gatk4/revertsam/main.nf index 3084658d..162aa0fa 100644 --- a/modules/gatk4/revertsam/main.nf +++ b/modules/gatk4/revertsam/main.nf @@ -43,7 +43,7 @@ process GATK4_REVERTSAM { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.reverted.bam + touch ${prefix}.bam cat <<-END_VERSIONS > versions.yml "${task.process}": From 08d5acbeb6c9e1b113ef3a0f4d32a6a1dbe522fe Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 11:59:08 +0200 Subject: [PATCH 45/91] feat added tests for stubs --- tests/modules/gatk4/mergebamalignment/main.nf | 11 +++++++++++ tests/modules/gatk4/mergebamalignment/test.yml | 9 +++++++++ tests/modules/gatk4/mutect2/main.nf | 18 ++++++++++++++++++ tests/modules/gatk4/mutect2/test.yml | 11 +++++++++++ tests/modules/gatk4/revertsam/main.nf | 8 ++++++++ tests/modules/gatk4/revertsam/test.yml | 9 +++++++++ tests/modules/gatk4/samtofastq/main.nf | 8 ++++++++ tests/modules/gatk4/samtofastq/test.yml | 10 ++++++++++ tests/modules/samtools/view/main.nf | 9 +++++++++ tests/modules/samtools/view/test.yml | 8 ++++++++ 10 files changed, 101 insertions(+) diff --git a/tests/modules/gatk4/mergebamalignment/main.nf b/tests/modules/gatk4/mergebamalignment/main.nf index 8a38c129..ebedad9b 100644 --- a/tests/modules/gatk4/mergebamalignment/main.nf +++ b/tests/modules/gatk4/mergebamalignment/main.nf @@ -14,3 +14,14 @@ workflow test_gatk4_mergebamalignment { GATK4_MERGEBAMALIGNMENT ( input, fasta, dict ) } + +workflow test_gatk4_mergebamalignment_stubs { + input = [ [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_unaligned_bam'], checkIfExists: true) + ] + fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + dict = file(params.test_data['sarscov2']['genome']['genome_dict'], checkIfExists: true) + + GATK4_MERGEBAMALIGNMENT ( input, fasta, dict ) +} diff --git a/tests/modules/gatk4/mergebamalignment/test.yml b/tests/modules/gatk4/mergebamalignment/test.yml index b1bb32b2..84a67654 100644 --- a/tests/modules/gatk4/mergebamalignment/test.yml +++ b/tests/modules/gatk4/mergebamalignment/test.yml @@ -7,3 +7,12 @@ - path: output/gatk4/test.bam md5sum: e6f1b343700b7ccb94e81ae127433988 - path: output/gatk4/versions.yml + +- name: gatk4 mergebamalignment test_gatk4_mergebamalignment_stubs + command: nextflow run ./tests/modules/gatk4/mergebamalignment -entry test_gatk4_mergebamalignment -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/mergebamalignment/nextflow.config -stub-run + tags: + - gatk4 + - gatk4/mergebamalignment + files: + - path: output/gatk4/test.bam + - path: output/gatk4/versions.yml diff --git a/tests/modules/gatk4/mutect2/main.nf b/tests/modules/gatk4/mutect2/main.nf index 0b4339f0..619d28b8 100644 --- a/tests/modules/gatk4/mutect2/main.nf +++ b/tests/modules/gatk4/mutect2/main.nf @@ -118,3 +118,21 @@ workflow test_gatk4_mutect2_mitochondria { GATK4_MUTECT2_MITO ( input, fasta, fai, dict, [], [], [], [] ) } + +workflow test_gatk4_mutect2_tumor_single_stubs { + input = [ [ id:'test'], // meta map + [ file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam'], checkIfExists: true)], + [ file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam_bai'], checkIfExists: true)], + [] + ] + + fasta = file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) + fai = file(params.test_data['homo_sapiens']['genome']['genome_21_fasta_fai'], checkIfExists: true) + dict = file(params.test_data['homo_sapiens']['genome']['genome_21_dict'], checkIfExists: true) + germline_resource = file(params.test_data['homo_sapiens']['genome']['gnomad_r2_1_1_21_vcf_gz'], checkIfExists: true) + germline_resource_tbi = file(params.test_data['homo_sapiens']['genome']['gnomad_r2_1_1_21_vcf_gz_tbi'], checkIfExists: true) + panel_of_normals = file(params.test_data['homo_sapiens']['genome']['mills_and_1000g_indels_21_vcf_gz'], checkIfExists: true) + panel_of_normals_tbi = file(params.test_data['homo_sapiens']['genome']['mills_and_1000g_indels_21_vcf_gz_tbi'], checkIfExists: true) + + GATK4_MUTECT2 ( input, fasta, fai, dict, germline_resource, germline_resource_tbi, panel_of_normals, panel_of_normals_tbi ) +} diff --git a/tests/modules/gatk4/mutect2/test.yml b/tests/modules/gatk4/mutect2/test.yml index 3cefce09..9232cedd 100644 --- a/tests/modules/gatk4/mutect2/test.yml +++ b/tests/modules/gatk4/mutect2/test.yml @@ -69,3 +69,14 @@ md5sum: fc6ea14ca2da346babe78161beea28c9 - path: output/gatk4/test.vcf.gz.tbi - path: output/gatk4/versions.yml + +- name: gatk4 mutect2 test_gatk4_mutect2_tumor_single_stubs + command: nextflow run ./tests/modules/gatk4/mutect2 -entry test_gatk4_mutect2_tumor_single -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/mutect2/nextflow.config -stub-run + tags: + - gatk4 + - gatk4/mutect2 + files: + - path: output/gatk4/test.vcf.gz + - path: output/gatk4/test.vcf.gz.stats + - path: output/gatk4/test.vcf.gz.tbi + - path: output/gatk4/versions.yml diff --git a/tests/modules/gatk4/revertsam/main.nf b/tests/modules/gatk4/revertsam/main.nf index ab5dddee..5b14d471 100644 --- a/tests/modules/gatk4/revertsam/main.nf +++ b/tests/modules/gatk4/revertsam/main.nf @@ -11,3 +11,11 @@ workflow test_gatk4_revertsam { GATK4_REVERTSAM ( input ) } + +workflow test_gatk4_revertsam_stubs { + input = [ [ id:'test' ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + + GATK4_REVERTSAM ( input ) +} diff --git a/tests/modules/gatk4/revertsam/test.yml b/tests/modules/gatk4/revertsam/test.yml index 2ebdb685..89e78659 100644 --- a/tests/modules/gatk4/revertsam/test.yml +++ b/tests/modules/gatk4/revertsam/test.yml @@ -7,3 +7,12 @@ - path: output/gatk4/test.reverted.bam md5sum: f783a88deb45c3a2c20ca12cbe1c5652 - path: output/gatk4/versions.yml + +- name: gatk4 revertsam test_gatk4_revertsam_stubs + command: nextflow run ./tests/modules/gatk4/revertsam -entry test_gatk4_revertsam -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/revertsam/nextflow.config -stub-run + tags: + - gatk4 + - gatk4/revertsam + files: + - path: output/gatk4/test.reverted.bam + - path: output/gatk4/versions.yml diff --git a/tests/modules/gatk4/samtofastq/main.nf b/tests/modules/gatk4/samtofastq/main.nf index 26a8ce2d..aad80057 100644 --- a/tests/modules/gatk4/samtofastq/main.nf +++ b/tests/modules/gatk4/samtofastq/main.nf @@ -19,3 +19,11 @@ workflow test_gatk4_samtofastq_paired_end { GATK4_SAMTOFASTQ ( input ) } + +workflow test_gatk4_samtofastq_paired_end_stubs { + input = [ [ id:'test', single_end: false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] + ] + + GATK4_SAMTOFASTQ ( input ) +} diff --git a/tests/modules/gatk4/samtofastq/test.yml b/tests/modules/gatk4/samtofastq/test.yml index eb25f33b..1288a270 100644 --- a/tests/modules/gatk4/samtofastq/test.yml +++ b/tests/modules/gatk4/samtofastq/test.yml @@ -19,3 +19,13 @@ - path: output/gatk4/test_2.fastq.gz md5sum: 613bf64c023609e1c62ad6ce9e4be8d7 - path: output/gatk4/versions.yml + +- name: gatk4 samtofastq test_gatk4_samtofastq_paired_end_stubs + command: nextflow run ./tests/modules/gatk4/samtofastq -entry test_gatk4_samtofastq_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/samtofastq/nextflow.config -stub-run + tags: + - gatk4 + - gatk4/samtofastq + files: + - path: output/gatk4/test_1.fastq.gz + - path: output/gatk4/test_2.fastq.gz + - path: output/gatk4/versions.yml diff --git a/tests/modules/samtools/view/main.nf b/tests/modules/samtools/view/main.nf index 9c239066..0e3f597e 100644 --- a/tests/modules/samtools/view/main.nf +++ b/tests/modules/samtools/view/main.nf @@ -22,3 +22,12 @@ workflow test_samtools_view_cram { SAMTOOLS_VIEW ( input, fasta ) } + +workflow test_samtools_view_stubs { + input = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true), + [] + ] + + SAMTOOLS_VIEW ( input, [] ) +} diff --git a/tests/modules/samtools/view/test.yml b/tests/modules/samtools/view/test.yml index 1287d455..2718130e 100644 --- a/tests/modules/samtools/view/test.yml +++ b/tests/modules/samtools/view/test.yml @@ -14,3 +14,11 @@ - samtools files: - path: output/samtools/test.cram + +- name: samtools view test_samtools_view_stubs + command: nextflow run ./tests/modules/samtools/view -entry test_samtools_view -c ./tests/config/nextflow.config -c ./tests/modules/samtools/view/nextflow.config -stub-run + tags: + - samtools/view + - samtools + files: + - path: output/samtools/test.bam From f231291e7730654158cbd3f10b82c292e27fa273 Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 12:58:12 +0200 Subject: [PATCH 46/91] fix gatk4_reversam test --- modules/gatk4/revertsam/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gatk4/revertsam/main.nf b/modules/gatk4/revertsam/main.nf index 162aa0fa..3084658d 100644 --- a/modules/gatk4/revertsam/main.nf +++ b/modules/gatk4/revertsam/main.nf @@ -43,7 +43,7 @@ process GATK4_REVERTSAM { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.bam + touch ${prefix}.reverted.bam cat <<-END_VERSIONS > versions.yml "${task.process}": From de40c1bf54b79efb4aa2396ded7f8f457cf33c02 Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 13:15:01 +0200 Subject: [PATCH 47/91] chore removing file from test --- tests/modules/gatk4/mergebamalignment/main.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/modules/gatk4/mergebamalignment/main.nf b/tests/modules/gatk4/mergebamalignment/main.nf index ebedad9b..0eb6876d 100644 --- a/tests/modules/gatk4/mergebamalignment/main.nf +++ b/tests/modules/gatk4/mergebamalignment/main.nf @@ -16,12 +16,12 @@ workflow test_gatk4_mergebamalignment { } workflow test_gatk4_mergebamalignment_stubs { - input = [ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true), - file(params.test_data['sarscov2']['illumina']['test_unaligned_bam'], checkIfExists: true) + input = [ [ id:'test' ], // meta map + "test_foo.bam", + "test_bar.bam" ] - fasta = file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) - dict = file(params.test_data['sarscov2']['genome']['genome_dict'], checkIfExists: true) + fasta = "genome.fasta" + dict = "genome.fasta.dict" GATK4_MERGEBAMALIGNMENT ( input, fasta, dict ) } From 3a4e415fe21982ccb39807d71c16e7d6ad0a1c1a Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 13:30:18 +0200 Subject: [PATCH 48/91] chores adding dummy files for stubs --- tests/modules/gatk4/mutect2/main.nf | 18 +++++++++--------- tests/modules/gatk4/revertsam/main.nf | 2 +- tests/modules/gatk4/samtofastq/main.nf | 2 +- tests/modules/samtools/view/main.nf | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/modules/gatk4/mutect2/main.nf b/tests/modules/gatk4/mutect2/main.nf index 619d28b8..251e1987 100644 --- a/tests/modules/gatk4/mutect2/main.nf +++ b/tests/modules/gatk4/mutect2/main.nf @@ -121,18 +121,18 @@ workflow test_gatk4_mutect2_mitochondria { workflow test_gatk4_mutect2_tumor_single_stubs { input = [ [ id:'test'], // meta map - [ file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam'], checkIfExists: true)], - [ file(params.test_data['homo_sapiens']['illumina']['test2_paired_end_recalibrated_sorted_bam_bai'], checkIfExists: true)], + [ "foo.bam" ], + [ "foo.bam.bai" ], [] ] - fasta = file(params.test_data['homo_sapiens']['genome']['genome_21_fasta'], checkIfExists: true) - fai = file(params.test_data['homo_sapiens']['genome']['genome_21_fasta_fai'], checkIfExists: true) - dict = file(params.test_data['homo_sapiens']['genome']['genome_21_dict'], checkIfExists: true) - germline_resource = file(params.test_data['homo_sapiens']['genome']['gnomad_r2_1_1_21_vcf_gz'], checkIfExists: true) - germline_resource_tbi = file(params.test_data['homo_sapiens']['genome']['gnomad_r2_1_1_21_vcf_gz_tbi'], checkIfExists: true) - panel_of_normals = file(params.test_data['homo_sapiens']['genome']['mills_and_1000g_indels_21_vcf_gz'], checkIfExists: true) - panel_of_normals_tbi = file(params.test_data['homo_sapiens']['genome']['mills_and_1000g_indels_21_vcf_gz_tbi'], checkIfExists: true) + fasta = "genome.fasta" + fai = "genome.fasta.fai" + dict = "genome.fasta.dict" + germline_resource = "genome_gnomAD.r2.1.1.vcf.gz" + germline_resource_tbi = "genome_gnomAD.r2.1.1.vcf.gz.tbi" + panel_of_normals = "genome_mills_and_1000G.indels.hg38.vcf.gz" + panel_of_normals_tbi = "genome_mills_and_1000G.indels.hg38.vcf.gz.tbi" GATK4_MUTECT2 ( input, fasta, fai, dict, germline_resource, germline_resource_tbi, panel_of_normals, panel_of_normals_tbi ) } diff --git a/tests/modules/gatk4/revertsam/main.nf b/tests/modules/gatk4/revertsam/main.nf index 5b14d471..738ecd8f 100644 --- a/tests/modules/gatk4/revertsam/main.nf +++ b/tests/modules/gatk4/revertsam/main.nf @@ -14,7 +14,7 @@ workflow test_gatk4_revertsam { workflow test_gatk4_revertsam_stubs { input = [ [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + "foo_paired_end.bam" ] GATK4_REVERTSAM ( input ) diff --git a/tests/modules/gatk4/samtofastq/main.nf b/tests/modules/gatk4/samtofastq/main.nf index aad80057..79d04c7c 100644 --- a/tests/modules/gatk4/samtofastq/main.nf +++ b/tests/modules/gatk4/samtofastq/main.nf @@ -22,7 +22,7 @@ workflow test_gatk4_samtofastq_paired_end { workflow test_gatk4_samtofastq_paired_end_stubs { input = [ [ id:'test', single_end: false ], // meta map - [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] + [ "foo_paired_end.bam" ] ] GATK4_SAMTOFASTQ ( input ) diff --git a/tests/modules/samtools/view/main.nf b/tests/modules/samtools/view/main.nf index 0e3f597e..bdad1078 100644 --- a/tests/modules/samtools/view/main.nf +++ b/tests/modules/samtools/view/main.nf @@ -25,7 +25,7 @@ workflow test_samtools_view_cram { workflow test_samtools_view_stubs { input = [ [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true), + "foo_paired_end.bam", [] ] From da82e06354c34bf6381a86e5af195b24d7ef98ee Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 14:16:07 +0200 Subject: [PATCH 49/91] fix gatk4_mutect2 test --- tests/modules/gatk4/mutect2/main.nf | 14 +++++++++----- tests/modules/gatk4/mutect2/test.yml | 5 +++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/modules/gatk4/mutect2/main.nf b/tests/modules/gatk4/mutect2/main.nf index 251e1987..486b78ad 100644 --- a/tests/modules/gatk4/mutect2/main.nf +++ b/tests/modules/gatk4/mutect2/main.nf @@ -119,10 +119,14 @@ workflow test_gatk4_mutect2_mitochondria { GATK4_MUTECT2_MITO ( input, fasta, fai, dict, [], [], [], [] ) } -workflow test_gatk4_mutect2_tumor_single_stubs { - input = [ [ id:'test'], // meta map - [ "foo.bam" ], - [ "foo.bam.bai" ], +workflow test_gatk4_mutect2_tumor_normal_pair_f1r2_stubs { + input = [ [ id:'test', normal_id:'normal', tumor_id:'tumour' ], // meta map + [ "foo_parired.bam", + "foo_parired2.bam" + ], + [ "foo_parired.bam.bai", + "foo_parired2.bam.bai" + ], [] ] @@ -134,5 +138,5 @@ workflow test_gatk4_mutect2_tumor_single_stubs { panel_of_normals = "genome_mills_and_1000G.indels.hg38.vcf.gz" panel_of_normals_tbi = "genome_mills_and_1000G.indels.hg38.vcf.gz.tbi" - GATK4_MUTECT2 ( input, fasta, fai, dict, germline_resource, germline_resource_tbi, panel_of_normals, panel_of_normals_tbi ) + GATK4_MUTECT2_F1R2 ( input, fasta, fai, dict, germline_resource, germline_resource_tbi, panel_of_normals, panel_of_normals_tbi ) } diff --git a/tests/modules/gatk4/mutect2/test.yml b/tests/modules/gatk4/mutect2/test.yml index 9232cedd..3853801d 100644 --- a/tests/modules/gatk4/mutect2/test.yml +++ b/tests/modules/gatk4/mutect2/test.yml @@ -70,12 +70,13 @@ - path: output/gatk4/test.vcf.gz.tbi - path: output/gatk4/versions.yml -- name: gatk4 mutect2 test_gatk4_mutect2_tumor_single_stubs - command: nextflow run ./tests/modules/gatk4/mutect2 -entry test_gatk4_mutect2_tumor_single -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/mutect2/nextflow.config -stub-run +- name: gatk4 mutect2 test_gatk4_mutect2_tumor_normal_pair_f1r2_stubs + command: nextflow run ./tests/modules/gatk4/mutect2 -entry test_gatk4_mutect2_tumor_normal_pair_f1r2 -c ./tests/config/nextflow.config -c ./tests/modules/gatk4/mutect2/nextflow.config -stub-run tags: - gatk4 - gatk4/mutect2 files: + - path: output/gatk4/test.f1r2.tar.gz - path: output/gatk4/test.vcf.gz - path: output/gatk4/test.vcf.gz.stats - path: output/gatk4/test.vcf.gz.tbi From ed4dd1a928ebf4308efb720de878045f7773f8e2 Mon Sep 17 00:00:00 2001 From: ljmesi <37740329+ljmesi@users.noreply.github.com> Date: Mon, 2 May 2022 14:16:12 +0200 Subject: [PATCH 50/91] Add additional handling of fq.gz file name suffix --- modules/metaphlan3/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/metaphlan3/main.nf b/modules/metaphlan3/main.nf index 3fc6b277..bff0eb9a 100644 --- a/modules/metaphlan3/main.nf +++ b/modules/metaphlan3/main.nf @@ -23,7 +23,7 @@ process METAPHLAN3 { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def input_type = ("$input".endsWith(".fastq.gz")) ? "--input_type fastq" : ("$input".contains(".fasta")) ? "--input_type fasta" : ("$input".endsWith(".bowtie2out.txt")) ? "--input_type bowtie2out" : "--input_type sam" + def input_type = ("$input".endsWith(".fastq.gz") || "$input".endsWith(".fq.gz")) ? "--input_type fastq" : ("$input".contains(".fasta")) ? "--input_type fasta" : ("$input".endsWith(".bowtie2out.txt")) ? "--input_type bowtie2out" : "--input_type sam" def input_data = ("$input_type".contains("fastq")) && !meta.single_end ? "${input[0]},${input[1]}" : "$input" def bowtie2_out = "$input_type" == "--input_type bowtie2out" || "$input_type" == "--input_type sam" ? '' : "--bowtie2out ${prefix}.bowtie2out.txt" From 5cd517e7596b8869d27c01505029ac1449067e95 Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Mon, 2 May 2022 14:31:40 +0200 Subject: [PATCH 51/91] Apply suggestions from code review Co-authored-by: James A. Fellows Yates --- modules/antismash/antismashlitedownloaddatabases/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/meta.yml b/modules/antismash/antismashlitedownloaddatabases/meta.yml index 4b9644c4..f7ddf3b0 100644 --- a/modules/antismash/antismashlitedownloaddatabases/meta.yml +++ b/modules/antismash/antismashlitedownloaddatabases/meta.yml @@ -27,7 +27,7 @@ input: - database_css: type: directory description: | - antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by ther use by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. + antismash/outputs/html/css folder which is being created during the antiSMASH database downloading step. These files are normally downloaded by download-antismash-databases itself, and must be retrieved by the user by manually running the command with conda or a standalone installation of antiSMASH. Therefore we do not recommend using this module for production pipelines, but rather require users to specify their own local copy of the antiSMASH database in pipelines. pattern: "css" - database_detection: type: directory From 0fc352188b8060331ee4268ee5492d9a54905b47 Mon Sep 17 00:00:00 2001 From: jasmezz Date: Mon, 2 May 2022 14:40:35 +0200 Subject: [PATCH 52/91] Fix conda antismash path --- modules/antismash/antismashlitedownloaddatabases/main.nf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index 5f9141f0..da8a750d 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -35,12 +35,19 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { script: def args = task.ext.args ?: '' + conda = params.enable_conda """ download-antismash-databases \\ --database-dir antismash_db \\ $args - cp -r /usr/local/lib/python3.8/site-packages/antismash antismash_dir + if [[ $conda = false ]]; \ + then \ + cp -r /usr/local/lib/python3.8/site-packages/antismash antismash_dir; \ + else \ + antismash_path=\$(python -c 'import antismash;print(antismash.__file__.split("__")[0])') \ + cp -r \$antismash_path antismash_dir; \ + fi cat <<-END_VERSIONS > versions.yml "${task.process}": From 62c6123ec48e15b42bd60b344603a83b658054d8 Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 15:11:57 +0200 Subject: [PATCH 53/91] fix gatk4_mutect2 test by changing main.nf --- modules/gatk4/mutect2/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/gatk4/mutect2/main.nf b/modules/gatk4/mutect2/main.nf index a214b57d..9969ad70 100644 --- a/modules/gatk4/mutect2/main.nf +++ b/modules/gatk4/mutect2/main.nf @@ -62,8 +62,8 @@ process GATK4_MUTECT2 { def prefix = task.ext.prefix ?: "${meta.id}" """ touch ${prefix}.vcf.gz - touch ${prefix}.tbi - touch ${prefix}.stats + touch ${prefix}.vcf.gz.tbi + touch ${prefix}.vcf.gz.stats touch ${prefix}.f1r2.tar.gz cat <<-END_VERSIONS > versions.yml From 50cc136a7810323f2c802ff3bcc1bf54c85d57a9 Mon Sep 17 00:00:00 2001 From: Lucpen Date: Mon, 2 May 2022 15:14:05 +0200 Subject: [PATCH 54/91] Update tests/modules/gatk4/mutect2/main.nf fix spelling mistake Co-authored-by: Maxime U. Garcia --- tests/modules/gatk4/mutect2/main.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/modules/gatk4/mutect2/main.nf b/tests/modules/gatk4/mutect2/main.nf index 486b78ad..310e9ca1 100644 --- a/tests/modules/gatk4/mutect2/main.nf +++ b/tests/modules/gatk4/mutect2/main.nf @@ -121,11 +121,11 @@ workflow test_gatk4_mutect2_mitochondria { workflow test_gatk4_mutect2_tumor_normal_pair_f1r2_stubs { input = [ [ id:'test', normal_id:'normal', tumor_id:'tumour' ], // meta map - [ "foo_parired.bam", - "foo_parired2.bam" + [ "foo_paired.bam", + "foo_paired2.bam" ], - [ "foo_parired.bam.bai", - "foo_parired2.bam.bai" + [ "foo_paired.bam.bai", + "foo_paired2.bam.bai" ], [] ] From 6a9f2077cf6f224c547e4c7ff0f703cedc69bda2 Mon Sep 17 00:00:00 2001 From: jasmezz Date: Mon, 2 May 2022 16:41:59 +0200 Subject: [PATCH 55/91] Fix unbound variable crash --- modules/antismash/antismashlitedownloaddatabases/main.nf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index da8a750d..fbb92490 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -37,16 +37,15 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { def args = task.ext.args ?: '' conda = params.enable_conda """ - download-antismash-databases \\ - --database-dir antismash_db \\ - $args + #download-antismash-databases \\ + # --database-dir antismash_db \\ + # $args if [[ $conda = false ]]; \ then \ cp -r /usr/local/lib/python3.8/site-packages/antismash antismash_dir; \ else \ - antismash_path=\$(python -c 'import antismash;print(antismash.__file__.split("__")[0])') \ - cp -r \$antismash_path antismash_dir; \ + cp -r \$(python -c 'import antismash;print(antismash.__file__.split("/__")[0])') antismash_dir; \ fi cat <<-END_VERSIONS > versions.yml From 8db0b754c0422cb80d06f47986d660f8aaf02c89 Mon Sep 17 00:00:00 2001 From: Jasmin F <73216762+jasmezz@users.noreply.github.com> Date: Mon, 2 May 2022 16:56:39 +0200 Subject: [PATCH 56/91] Fix commented-out command --- modules/antismash/antismashlitedownloaddatabases/main.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/antismash/antismashlitedownloaddatabases/main.nf b/modules/antismash/antismashlitedownloaddatabases/main.nf index fbb92490..a0928333 100644 --- a/modules/antismash/antismashlitedownloaddatabases/main.nf +++ b/modules/antismash/antismashlitedownloaddatabases/main.nf @@ -37,9 +37,9 @@ process ANTISMASH_ANTISMASHLITEDOWNLOADDATABASES { def args = task.ext.args ?: '' conda = params.enable_conda """ - #download-antismash-databases \\ - # --database-dir antismash_db \\ - # $args + download-antismash-databases \\ + --database-dir antismash_db \\ + $args if [[ $conda = false ]]; \ then \ From 24dc2d21132b07b4c3d36c9836885427a5019b11 Mon Sep 17 00:00:00 2001 From: jvhagey Date: Mon, 2 May 2022 11:30:19 -0400 Subject: [PATCH 57/91] added tests --- modules/srst2/srst2/main.nf | 4 ++-- modules/srst2/srst2/meta.yml | 6 +++++- tests/modules/srst2/srst2/main.nf | 31 +++++++++++++++++++++++++++--- tests/modules/srst2/srst2/test.yml | 21 ++++++++++++++++++-- 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/modules/srst2/srst2/main.nf b/modules/srst2/srst2/main.nf index 4ee9f6a6..e8a91716 100644 --- a/modules/srst2/srst2/main.nf +++ b/modules/srst2/srst2/main.nf @@ -11,8 +11,9 @@ process SRST2_SRST2 { tuple val(meta), path(fastq_s), path(db) output: - tuple val(meta), path("*_genes_*_results.txt") , emit: gene_results + tuple val(meta), path("*_genes_*_results.txt") , optional:true, emit: gene_results tuple val(meta), path("*_fullgenes_*_results.txt") , optional:true, emit: fullgene_results + tuple val(meta), path("*_mlst_*_results.txt") , optional:true, emit: mlst_results tuple val(meta), path("*.pileup") , emit: pileup tuple val(meta), path("*.sorted.bam") , emit: sorted_bam path "versions.yml" , emit: versions @@ -38,7 +39,6 @@ process SRST2_SRST2 { --output ${prefix} \\ ${database} \\ $args - cat <<-END_VERSIONS > versions.yml "${task.process}": srst2: \$(echo \$(srst2 --version 2>&1) | sed 's/srst2 //' )) diff --git a/modules/srst2/srst2/meta.yml b/modules/srst2/srst2/meta.yml index 3755fb34..94c763bb 100644 --- a/modules/srst2/srst2/meta.yml +++ b/modules/srst2/srst2/meta.yml @@ -49,12 +49,16 @@ output: pattern: "versions.yml" - txt: type: file - description: A detailed report, with one row per gene per sample described here: https://github.com/katholt/srst2#gene-typing + description: A detailed report, with one row per gene per sample described here github.com/katholt/srst2#gene-typing pattern: "*_fullgenes_*_results.txt" - txt: type: file description: A tabulated summary report of samples x genes. pattern: "*_genes_*_results.txt" + - txt: + type: file + description: A tabulated summary report of mlst subtyping. + pattern: "*_mlst_*_results.txt" - bam: type: file description: Sorted BAM file diff --git a/tests/modules/srst2/srst2/main.nf b/tests/modules/srst2/srst2/main.nf index 235f3ff9..7d51937b 100644 --- a/tests/modules/srst2/srst2/main.nf +++ b/tests/modules/srst2/srst2/main.nf @@ -4,9 +4,34 @@ nextflow.enable.dsl = 2 include { SRST2_SRST2 } from '../../../../modules/srst2/srst2/main.nf' +workflow test_srst2_srst2_exit { + + input = [ + [ id:'test', single_end:false, db:"test"], // meta map + [ file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true), + file(params.test_data['bacteroides_fragilis']['illumina']['test1_2_fastq_gz'], checkIfExists: true) ], +// [("")] + file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/resFinder_20180221_srst2.fasta') + ] + + SRST2_SRST2(input) +} + +workflow test_srst2_srst2_mlst { + + input = [ + [ id:'test', single_end:false, db:"mlst"], // meta map + [ file("https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/SRR9067271_1.fastq.gz", checkIfExists: true), + file("https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/SRR9067271_2.fastq.gz", checkIfExists: true) ], + file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/MLST_DB.fas') + ] + + SRST2_SRST2(input) +} + workflow test_srst2_srst2_paired_end { - input = [ + input = [ [ id:'test', single_end:false, db:"gene"], // meta map [ file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true), file(params.test_data['bacteroides_fragilis']['illumina']['test1_2_fastq_gz'], checkIfExists: true) ], @@ -18,11 +43,11 @@ workflow test_srst2_srst2_paired_end { workflow test_srst2_srst2_single_end { - input = [ + input = [ [ id:'test', single_end:true, db:"gene" ], // meta map file(params.test_data['bacteroides_fragilis']['illumina']['test1_1_fastq_gz'], checkIfExists: true), file('https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/delete_me/srst2/resFinder_20180221_srst2.fasta') // Change to params.test_data syntax after the data is included in tests/config/test_data.config ] SRST2_SRST2(input) -} \ No newline at end of file +} diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index f011b4d1..102c78a4 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -1,8 +1,22 @@ +- name: srst2 srst2 test_srst2_srst2_mlst + command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_mlst -c tests/config/nextflow.config + tags: + - srst2/srst2 + - srst2 + files: + - path: output/srst2/test__SRR9067271.MLST_DB.pileup + md5sum: f59217dd9340264b9913c20b545b2ce7 + - path: output/srst2/test__SRR9067271.MLST_DB.sorted.bam + - path: output/srst2/test__mlst__MLST_DB__results.txt + md5sum: ec1b1f69933401d67c57f64cad11a098 + - path: output/srst2/versions.yml + md5sum: a0c256a2fd3636069710b8ef22ee5ea7 + - name: srst2 srst2 test_srst2_srst2_paired_end command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_paired_end -c tests/config/nextflow.config tags: - - srst2 - srst2/srst2 + - srst2 files: - path: output/srst2/test__genes__resFinder_20180221_srst2__results.txt md5sum: 099aa6cacec5524b311f606debdfb3a9 @@ -15,8 +29,8 @@ - name: srst2 srst2 test_srst2_srst2_single_end command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_single_end -c tests/config/nextflow.config tags: - - srst2 - srst2/srst2 + - srst2 files: - path: output/srst2/test__fullgenes__resFinder_20180221_srst2__results.txt md5sum: d0762ef8c38afd0e0a34cce52ed1a3db @@ -27,3 +41,6 @@ - path: output/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam - path: output/srst2/versions.yml md5sum: 790fe00493c6634d17801a930073218b + +- name: srst2 srst2 test_srst2_srst2_exit #Testing pipeline exit when not meta.db + exit_code: 1 From cd45dc15503972b3f4bce572fe98168f68e66f9e Mon Sep 17 00:00:00 2001 From: jvhagey Date: Mon, 2 May 2022 11:38:39 -0400 Subject: [PATCH 58/91] linting test.yml --- tests/modules/srst2/srst2/test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 102c78a4..9e448211 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -1,3 +1,9 @@ +- name: srst2 srst2 test_srst2_srst2_exit #Testing pipeline exit when not meta.db + tags: + - srst2/srst2 + - srst2 + exit_code: 1 + - name: srst2 srst2 test_srst2_srst2_mlst command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_mlst -c tests/config/nextflow.config tags: @@ -41,6 +47,3 @@ - path: output/srst2/test__test1_1.resFinder_20180221_srst2.sorted.bam - path: output/srst2/versions.yml md5sum: 790fe00493c6634d17801a930073218b - -- name: srst2 srst2 test_srst2_srst2_exit #Testing pipeline exit when not meta.db - exit_code: 1 From 99a87119970762fea09f8c71080dabf88d0836f3 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Mon, 2 May 2022 12:26:56 -0400 Subject: [PATCH 59/91] Update tests/modules/srst2/srst2/test.yml Co-authored-by: Edmund Miller --- tests/modules/srst2/srst2/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 9e448211..5ed236d3 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -1,4 +1,5 @@ - name: srst2 srst2 test_srst2_srst2_exit #Testing pipeline exit when not meta.db + command: nextflow run tests/modules/srst2/srst2 -entry test_srst2_srst2_exit -c tests/config/nextflow.config tags: - srst2/srst2 - srst2 From 16aa3915e3853c6baed6d847dedc0229340c7901 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 2 May 2022 23:16:59 +0200 Subject: [PATCH 60/91] feat: add module for slimfastq --- modules/slimfastq/main.nf | 52 +++++++++++++++++++++++++ modules/slimfastq/meta.yml | 41 +++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/slimfastq/main.nf | 46 ++++++++++++++++++++++ tests/modules/slimfastq/nextflow.config | 5 +++ tests/modules/slimfastq/test.yml | 41 +++++++++++++++++++ 6 files changed, 189 insertions(+) create mode 100644 modules/slimfastq/main.nf create mode 100644 modules/slimfastq/meta.yml create mode 100644 tests/modules/slimfastq/main.nf create mode 100644 tests/modules/slimfastq/nextflow.config create mode 100644 tests/modules/slimfastq/test.yml diff --git a/modules/slimfastq/main.nf b/modules/slimfastq/main.nf new file mode 100644 index 00000000..19ca876f --- /dev/null +++ b/modules/slimfastq/main.nf @@ -0,0 +1,52 @@ +def VERSION = '2.04' + +process SLIMFASTQ { + tag "$meta.id" + label 'process_low' + + conda (params.enable_conda ? "bioconda::slimfastq=2.04" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/slimfastq:2.04--h87f3376_2': + 'quay.io/biocontainers/slimfastq:2.04--h87f3376_2' }" + + input: + tuple val(meta), path(fastq) + + output: + tuple val(meta), path("*.sfq"), emit: sfq + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + if (meta.single_end) { + """ + gzip -d -c '${fastq}' | slimfastq \\ + $args \\ + -f '${prefix}.sfq' + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + slimfastq: ${VERSION} + END_VERSIONS + """ + } else { + """ + gzip -d -c '${fastq[0]}' | slimfastq \\ + $args \\ + -f '${prefix}_1.sfq' + + gzip -d -c '${fastq[1]}' | slimfastq \\ + $args \\ + -f '${prefix}_2.sfq' + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + slimfastq: ${VERSION} + END_VERSIONS + """ + } +} diff --git a/modules/slimfastq/meta.yml b/modules/slimfastq/meta.yml new file mode 100644 index 00000000..3b040f25 --- /dev/null +++ b/modules/slimfastq/meta.yml @@ -0,0 +1,41 @@ +name: "slimfastq" +description: Fast, efficient, lossless compression of FASTQ files. +keywords: + - FASTQ + - compression + - lossless +tools: + - "slimfastq": + description: "slimfastq efficiently compresses/decompresses FASTQ files" + homepage: "https://github.com/Infinidat/slimfastq" + tool_dev_url: "https://github.com/Infinidat/slimfastq" + licence: "['BSD-3-clause']" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - fastq: + type: file + description: Either a single-end FASTQ file or paired-end files. + pattern: "*.{fq.gz,fastq.gz}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - sfq: + type: file + description: Either one or two sequence files in slimfastq compressed format. + pattern: "*.{sfq}" + +authors: + - "@Midnighter" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index dcf85ba0..9da057d9 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -1727,6 +1727,10 @@ sistr: - modules/sistr/** - tests/modules/sistr/** +slimfastq: + - modules/slimfastq/** + - tests/modules/slimfastq/** + snapaligner/index: - modules/snapaligner/index/** - tests/modules/snapaligner/index/** diff --git a/tests/modules/slimfastq/main.nf b/tests/modules/slimfastq/main.nf new file mode 100644 index 00000000..40d44285 --- /dev/null +++ b/tests/modules/slimfastq/main.nf @@ -0,0 +1,46 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { SLIMFASTQ } from '../../../modules/slimfastq/main.nf' + +workflow test_slimfastq_single_end { + + input = [ + [ id:'test', single_end:true ], // meta map + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + + SLIMFASTQ ( input ) +} + +workflow test_slimfastq_paired_end { + + input = [ + [ id:'test', single_end:false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true), + file(params.test_data['sarscov2']['illumina']['test_2_fastq_gz'], checkIfExists: true)] + ] + + SLIMFASTQ ( input ) +} + +workflow test_slimfastq_nanopore { + + input = [ + [ id:'test', single_end:true ], // meta map + file(params.test_data['sarscov2']['nanopore']['test_fastq_gz'], checkIfExists: true) + ] + + SLIMFASTQ ( input ) +} + +workflow test_slimfastq_pacbio { + + input = [ + [ id:'test', single_end:true ], // meta map + file(params.test_data['homo_sapiens']['pacbio']['ccs_fq_gz'], checkIfExists: true) + ] + + SLIMFASTQ ( input ) +} diff --git a/tests/modules/slimfastq/nextflow.config b/tests/modules/slimfastq/nextflow.config new file mode 100644 index 00000000..50f50a7a --- /dev/null +++ b/tests/modules/slimfastq/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/slimfastq/test.yml b/tests/modules/slimfastq/test.yml new file mode 100644 index 00000000..f0d8b446 --- /dev/null +++ b/tests/modules/slimfastq/test.yml @@ -0,0 +1,41 @@ +- name: slimfastq test_slimfastq_single_end + command: nextflow run tests/modules/slimfastq -entry test_slimfastq_single_end -c tests/config/nextflow.config + tags: + - slimfastq + files: + - path: output/slimfastq/test.sfq + md5sum: 6a942eeca6c99ee9a9a0cedab5d246f1 + - path: output/slimfastq/versions.yml + md5sum: f52351f5c9e6259af02745c8eae5c780 + +- name: slimfastq test_slimfastq_paired_end + command: nextflow run tests/modules/slimfastq -entry test_slimfastq_paired_end -c tests/config/nextflow.config + tags: + - slimfastq + files: + - path: output/slimfastq/test_1.sfq + md5sum: 6a942eeca6c99ee9a9a0cedab5d246f1 + - path: output/slimfastq/test_2.sfq + md5sum: 0d2c60b52a39f7c2cb7843e848d90afd + - path: output/slimfastq/versions.yml + md5sum: 6239853705877651a4851c4cb6d62da4 + +- name: slimfastq test_slimfastq_nanopore + command: nextflow run tests/modules/slimfastq -entry test_slimfastq_nanopore -c tests/config/nextflow.config + tags: + - slimfastq + files: + - path: output/slimfastq/test.sfq + md5sum: e17f14d64d3a75356b03ff2f9e8881f7 + - path: output/slimfastq/versions.yml + md5sum: 33153f1103482a2bd35cb2f4c337c5e8 + +- name: slimfastq test_slimfastq_pacbio + command: nextflow run tests/modules/slimfastq -entry test_slimfastq_pacbio -c tests/config/nextflow.config + tags: + - slimfastq + files: + - path: output/slimfastq/test.sfq + md5sum: 9e8389e47e6ddf8c25e92412dd628339 + - path: output/slimfastq/versions.yml + md5sum: 1982789c3d5c7de37c0a9351e4ae63f7 From e5d7c80bb50dc58c360c3234faa243964d1a2dbb Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Mon, 2 May 2022 18:13:14 -0400 Subject: [PATCH 61/91] Update test.yml --- tests/modules/srst2/srst2/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 5ed236d3..9ea38dd6 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -12,11 +12,11 @@ - srst2 files: - path: output/srst2/test__SRR9067271.MLST_DB.pileup - md5sum: f59217dd9340264b9913c20b545b2ce7 + contains: "dnaJ_1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" - path: output/srst2/test__SRR9067271.MLST_DB.sorted.bam - path: output/srst2/test__mlst__MLST_DB__results.txt md5sum: ec1b1f69933401d67c57f64cad11a098 - - path: output/srst2/versions.yml + - path: /tmp/tmp74070b6r/srst2/versions.yml md5sum: a0c256a2fd3636069710b8ef22ee5ea7 - name: srst2 srst2 test_srst2_srst2_paired_end From 7b0d6f96fd0764254510df99e238be33b0acf862 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Mon, 2 May 2022 18:20:54 -0400 Subject: [PATCH 62/91] Update test.yml --- tests/modules/srst2/srst2/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 9ea38dd6..2b0c0d70 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -12,7 +12,8 @@ - srst2 files: - path: output/srst2/test__SRR9067271.MLST_DB.pileup - contains: "dnaJ_1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" + contains: + - "dnaJ_1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" - path: output/srst2/test__SRR9067271.MLST_DB.sorted.bam - path: output/srst2/test__mlst__MLST_DB__results.txt md5sum: ec1b1f69933401d67c57f64cad11a098 From db9681029c7a75da227577e29f26cb26de78a2e5 Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Mon, 2 May 2022 18:27:01 -0400 Subject: [PATCH 63/91] Update test.yml --- tests/modules/srst2/srst2/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 2b0c0d70..12b2a4f0 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -13,11 +13,11 @@ files: - path: output/srst2/test__SRR9067271.MLST_DB.pileup contains: - - "dnaJ_1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" + - "dnaJ-1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" - path: output/srst2/test__SRR9067271.MLST_DB.sorted.bam - path: output/srst2/test__mlst__MLST_DB__results.txt md5sum: ec1b1f69933401d67c57f64cad11a098 - - path: /tmp/tmp74070b6r/srst2/versions.yml + - path: output/srst2/versions.yml md5sum: a0c256a2fd3636069710b8ef22ee5ea7 - name: srst2 srst2 test_srst2_srst2_paired_end From 23cfb5dd42dc3dba37ee4588cf1262ce5c59ef6e Mon Sep 17 00:00:00 2001 From: "Jill V. Hagey, PhD" Date: Mon, 2 May 2022 18:38:48 -0400 Subject: [PATCH 64/91] Update test.yml --- tests/modules/srst2/srst2/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/srst2/srst2/test.yml b/tests/modules/srst2/srst2/test.yml index 12b2a4f0..f7621f4f 100644 --- a/tests/modules/srst2/srst2/test.yml +++ b/tests/modules/srst2/srst2/test.yml @@ -13,7 +13,7 @@ files: - path: output/srst2/test__SRR9067271.MLST_DB.pileup contains: - - "dnaJ-1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" + - "dnaJ-1 2 C 17 .........,....... FFFFFFFFFFFFFFFFF" - path: output/srst2/test__SRR9067271.MLST_DB.sorted.bam - path: output/srst2/test__mlst__MLST_DB__results.txt md5sum: ec1b1f69933401d67c57f64cad11a098 From b8a59c2b173cf0d987effc8a3e1cd547307a6085 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 08:20:10 +0200 Subject: [PATCH 65/91] Fixed version annotation --- modules/vardictjava/main.nf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 08318c29..682c26be 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -24,6 +24,8 @@ process VARDICTJAVA { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" + def VERSION = '1.8.3' + """ vardict-java \\ $args \\ @@ -41,7 +43,8 @@ process VARDICTJAVA { cat <<-END_VERSIONS > versions.yml "${task.process}": - vardict-java: \$(echo 1.8.3) + vardict-java: \$VERSION + var2vcf_valid.pl: \$(echo \$(var2vcf_valid.pl -h | sed -n 2p | awk '{ print \$2 }')) END_VERSIONS """ } From 9a10e5beb56904061e0b728fea55c8246af74997 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 08:29:46 +0200 Subject: [PATCH 66/91] Added the missing inputs in meta.yml + correct notation --- modules/vardictjava/main.nf | 10 +++++----- modules/vardictjava/meta.yml | 16 ++++++++++++++-- tests/modules/vardictjava/nextflow.config | 4 ---- tests/modules/vardictjava/test.yml | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 682c26be..14964989 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -9,8 +9,8 @@ process VARDICTJAVA { input: tuple val(meta), path(bam), path(bai) - path(regions_of_interest) - tuple path(reference_fasta), path(reference_fai) + path(bed) + tuple path(fasta), path(fasta_fai) output: tuple val(meta), path("*.vcf.gz"), emit: vcf @@ -33,8 +33,8 @@ process VARDICTJAVA { -b $bam \\ -th $task.cpus \\ -N $prefix \\ - -G $reference_fasta \\ - $regions_of_interest \\ + -G $fasta \\ + $bed \\ | teststrandbias.R \\ | var2vcf_valid.pl \\ $args2 \\ @@ -43,7 +43,7 @@ process VARDICTJAVA { cat <<-END_VERSIONS > versions.yml "${task.process}": - vardict-java: \$VERSION + vardict-java: $VERSION var2vcf_valid.pl: \$(echo \$(var2vcf_valid.pl -h | sed -n 2p | awk '{ print \$2 }')) END_VERSIONS """ diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index e3b2efe7..42480bc1 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -25,17 +25,29 @@ input: type: file description: BAM/SAM file pattern: "*.{bam,sam}" + + - bai: + type: file + description: Index of the BAM file + pattern: "*.bai" - - reference_fasta: + - fasta: type: file description: FASTA of the reference genome pattern: "*.{fa,fasta}" - - regions_of_interest: + - fasta_fai: + type: file + description: The index of the FASTA of the reference genome + pattern: "*.fai" + + - bed: type: file description: BED with the regions of interest pattern: "*.bed" + + output: - meta: type: map diff --git a/tests/modules/vardictjava/nextflow.config b/tests/modules/vardictjava/nextflow.config index e08201cc..50f50a7a 100644 --- a/tests/modules/vardictjava/nextflow.config +++ b/tests/modules/vardictjava/nextflow.config @@ -2,8 +2,4 @@ process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: VARDICTJAVA { - ext.args = '' - ext.args2 = '' - } } \ No newline at end of file diff --git a/tests/modules/vardictjava/test.yml b/tests/modules/vardictjava/test.yml index 549d688e..8cb29c4e 100644 --- a/tests/modules/vardictjava/test.yml +++ b/tests/modules/vardictjava/test.yml @@ -6,4 +6,4 @@ - path: output/vardictjava/test.vcf.gz md5sum: 3f1f227afc532bddeb58f16fd3013fc8 - path: output/vardictjava/versions.yml - md5sum: aac455b8c9c9194c5fed80e4fd495b96 + md5sum: 9b62c431a4f2680412b61c7071bdb1cd From 71c5d213742b56c62a75b6f34b1f734e073f415e Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 08:34:12 +0200 Subject: [PATCH 67/91] ran prettier --- modules/vardictjava/meta.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 42480bc1..83675fdc 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -25,10 +25,10 @@ input: type: file description: BAM/SAM file pattern: "*.{bam,sam}" - + - bai: type: file - description: Index of the BAM file + description: Index of the BAM file pattern: "*.bai" - fasta: @@ -46,8 +46,6 @@ input: description: BED with the regions of interest pattern: "*.bed" - - output: - meta: type: map From d33253513fcd8900db9480450496886ea1f17adb Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 10:15:30 +0200 Subject: [PATCH 68/91] put the version on top of the file --- modules/vardictjava/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 14964989..454b86a4 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -1,3 +1,5 @@ +def VERSION = '1.8.3' + process VARDICTJAVA { tag "$meta.id" label 'process_medium' @@ -24,8 +26,6 @@ process VARDICTJAVA { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def VERSION = '1.8.3' - """ vardict-java \\ $args \\ From 7c89af359c19131d2f9e8d32bd6ebb0dd682e4d2 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:15:51 +0200 Subject: [PATCH 69/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 83675fdc..63a52611 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -20,7 +20,6 @@ input: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - bam: type: file description: BAM/SAM file From 2dff193b543d78c1176d19c6f9c42385caa131c2 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:15:59 +0200 Subject: [PATCH 70/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 63a52611..a0af9fcc 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -29,7 +29,6 @@ input: type: file description: Index of the BAM file pattern: "*.bai" - - fasta: type: file description: FASTA of the reference genome From 3cdbdf50fc6423826e4aeed8fef8e1f75a19f621 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:16:14 +0200 Subject: [PATCH 71/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index a0af9fcc..3afa7e62 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -24,7 +24,6 @@ input: type: file description: BAM/SAM file pattern: "*.{bam,sam}" - - bai: type: file description: Index of the BAM file From 1609dfc96dd1ff205f1cac511b8dabb64f8a3db6 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:16:20 +0200 Subject: [PATCH 72/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 3afa7e62..325ce1ca 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -32,7 +32,6 @@ input: type: file description: FASTA of the reference genome pattern: "*.{fa,fasta}" - - fasta_fai: type: file description: The index of the FASTA of the reference genome From 01085b3e090b0e64ed0db23a93e00bd891a11639 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:16:30 +0200 Subject: [PATCH 73/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 325ce1ca..5be27690 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -36,7 +36,6 @@ input: type: file description: The index of the FASTA of the reference genome pattern: "*.fai" - - bed: type: file description: BED with the regions of interest From 11c6faf433d19803a1a9c198c83bde5ce5ae9706 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:16:36 +0200 Subject: [PATCH 74/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 5be27690..196b090e 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -47,7 +47,6 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] - - versions: type: file description: File containing software versions From 818764128039936e8d94df70427667638b0276a9 Mon Sep 17 00:00:00 2001 From: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> Date: Tue, 3 May 2022 10:16:45 +0200 Subject: [PATCH 75/91] Update modules/vardictjava/meta.yml Co-authored-by: James A. Fellows Yates --- modules/vardictjava/meta.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/vardictjava/meta.yml b/modules/vardictjava/meta.yml index 196b090e..59fba966 100644 --- a/modules/vardictjava/meta.yml +++ b/modules/vardictjava/meta.yml @@ -51,7 +51,6 @@ output: type: file description: File containing software versions pattern: "versions.yml" - - vcf: type: file description: VCF file output From 768073be6390c620e133230dea376466383832f5 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 11:46:19 +0100 Subject: [PATCH 76/91] bowtie2|Added support for large indexes --- modules/bowtie2/align/main.nf | 4 ++ tests/modules/bowtie2/align/main.nf | 2 +- tests/modules/bowtie2/align/nextflow.config | 13 ++++++- tests/modules/bowtie2/align/test.yml | 43 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 20b08f72..e68dbf24 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -26,6 +26,8 @@ process BOWTIE2_ALIGN { def unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : '' """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` + [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` + [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 bowtie2 \\ -x \$INDEX \\ -U $reads \\ @@ -46,6 +48,8 @@ process BOWTIE2_ALIGN { def unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : '' """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` + [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` + [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 bowtie2 \\ -x \$INDEX \\ -1 ${reads[0]} \\ diff --git a/tests/modules/bowtie2/align/main.nf b/tests/modules/bowtie2/align/main.nf index 2d3945e5..f91394ff 100644 --- a/tests/modules/bowtie2/align/main.nf +++ b/tests/modules/bowtie2/align/main.nf @@ -32,4 +32,4 @@ workflow test_bowtie2_align_paired_end { BOWTIE2_BUILD ( fasta ) BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned ) -} +} \ No newline at end of file diff --git a/tests/modules/bowtie2/align/nextflow.config b/tests/modules/bowtie2/align/nextflow.config index 8730f1c4..b4640de7 100644 --- a/tests/modules/bowtie2/align/nextflow.config +++ b/tests/modules/bowtie2/align/nextflow.config @@ -1,5 +1,16 @@ +params { + force_large_index = false +} + process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - +} + +if (params.force_large_index) { + process { + withName: BOWTIE2_BUILD { + ext.args = '--large-index' + } + } } diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 95d48b88..11e4c4e5 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -39,3 +39,46 @@ md5sum: 52be6950579598a990570fbcf5372184 - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 md5sum: e3b4ef343dea4dd571642010a7d09597 + +- name: bowtie2 align single-end large-index + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index + tags: + tags: + - bowtie2 + - bowtie2/align + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/bowtie2/bowtie2/genome.3.bt2 + md5sum: 8952b3e0b1ce9a7a5916f2e147180853 + - path: ./output/bowtie2/bowtie2/genome.2.bt2 + md5sum: 22c284084784a0720989595e0c9461fd + - path: ./output/bowtie2/bowtie2/genome.1.bt2 + md5sum: 07d811cd4e350d56267183d2ac7023a5 + - path: ./output/bowtie2/bowtie2/genome.4.bt2 + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 + md5sum: fda48e35925fb24d1c0785f021981e25 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 + md5sum: 802c26d32b970e1b105032b7ce7348b4 + +- name: bowtie2 align paired-end large-index + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index + tags: + - bowtie2 + - bowtie2/align + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/bowtie2/bowtie2/genome.3.bt2l + md5sum: 8952b3e0b1ce9a7a5916f2e147180853 + - path: ./output/bowtie2/bowtie2/genome.2.bt2l + md5sum: 22c284084784a0720989595e0c9461fd + - path: ./output/bowtie2/bowtie2/genome.1.bt2l + md5sum: 07d811cd4e350d56267183d2ac7023a5 + - path: ./output/bowtie2/bowtie2/genome.4.bt2l + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l + md5sum: fda48e35925fb24d1c0785f021981e25 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l + md5sum: 802c26d32b970e1b105032b7ce7348b4 From 0f1ee7e50dccd1a8717cd448c518b67fb68c1d09 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 12:04:51 +0100 Subject: [PATCH 77/91] bowtie2|test fixes --- tests/modules/bowtie2/align/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 11e4c4e5..961009b4 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -49,17 +49,17 @@ files: - path: ./output/bowtie2/test.bam - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2 + - path: ./output/bowtie2/bowtie2/genome.3.bt2l md5sum: 8952b3e0b1ce9a7a5916f2e147180853 - - path: ./output/bowtie2/bowtie2/genome.2.bt2 + - path: ./output/bowtie2/bowtie2/genome.2.bt2l md5sum: 22c284084784a0720989595e0c9461fd - - path: ./output/bowtie2/bowtie2/genome.1.bt2 + - path: ./output/bowtie2/bowtie2/genome.1.bt2l md5sum: 07d811cd4e350d56267183d2ac7023a5 - - path: ./output/bowtie2/bowtie2/genome.4.bt2 + - path: ./output/bowtie2/bowtie2/genome.4.bt2l md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l md5sum: fda48e35925fb24d1c0785f021981e25 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l md5sum: 802c26d32b970e1b105032b7ce7348b4 - name: bowtie2 align paired-end large-index From 40e6bd9dc16a7a4b63d399cf79f0bb1b7e3927c0 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 12:07:04 +0100 Subject: [PATCH 78/91] bowtie2|fix tags error --- tests/modules/bowtie2/align/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 961009b4..ea9aa72a 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -42,7 +42,6 @@ - name: bowtie2 align single-end large-index command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index - tags: tags: - bowtie2 - bowtie2/align From 96d4a31cc15e3b20ee6a7df0ad39ca59feeb3a02 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 15:31:05 +0200 Subject: [PATCH 79/91] Moved the BED file to the main tuple --- modules/vardictjava/main.nf | 3 +-- tests/modules/vardictjava/main.nf | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 454b86a4..c99301bb 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -10,8 +10,7 @@ process VARDICTJAVA { 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - tuple val(meta), path(bam), path(bai) - path(bed) + tuple val(meta), path(bam), path(bai), path(bed) tuple path(fasta), path(fasta_fai) output: diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index d392358a..e29490f3 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -9,15 +9,14 @@ workflow test_vardictjava { bam_input_ch = Channel.value([ [ id:'test' ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) ]) - bed = Channel.value(file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true)) - reference = Channel.value([ file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) ]) - VARDICTJAVA ( bam_input_ch, bed, reference ) + VARDICTJAVA ( bam_input_ch, reference ) } From c517ce9b4d3f09f2d49cf0c6795019a6e78938b3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 13:42:12 +0200 Subject: [PATCH 80/91] Added the module and some simple testing --- modules/happy/main.nf | 53 +++++++++++++++++++++++++++++ modules/happy/meta.yml | 51 +++++++++++++++++++++++++++ tests/config/pytest_modules.yml | 4 +++ tests/modules/happy/main.nf | 39 +++++++++++++++++++++ tests/modules/happy/nextflow.config | 5 +++ tests/modules/happy/test.yml | 7 ++++ 6 files changed, 159 insertions(+) create mode 100644 modules/happy/main.nf create mode 100644 modules/happy/meta.yml create mode 100644 tests/modules/happy/main.nf create mode 100644 tests/modules/happy/nextflow.config create mode 100644 tests/modules/happy/test.yml diff --git a/modules/happy/main.nf b/modules/happy/main.nf new file mode 100644 index 00000000..68059dd8 --- /dev/null +++ b/modules/happy/main.nf @@ -0,0 +1,53 @@ +def VERSION_HAP = '0.3.14' +def VERSION_PRE = '0.3.14' + +process HAPPY { + tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? "bioconda::hap.py=0.3.14" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/hap.py:0.3.14--py27h5c5a3ab_0': + 'quay.io/biocontainers/hap.py:0.3.14--py27h5c5a3ab_0' }" + + input: + tuple val(meta), path(truth_vcf), path(query_vcf), path(bed) + tuple path(fasta), path(fasta_fai) + + output: + tuple val(meta), path('*.csv'), path('*.json') , emit: metrics + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def args2 = task.ext.args2 ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + pre.py \\ + $args \\ + -R $bed \\ + --reference $fasta \\ + --threads $task.cpus \\ + $query_vcf \\ + ${prefix}_preprocessed.vcf.gz + + hap.py \\ + $truth_vcf \\ + ${prefix}_preprocessed.vcf.gz \\ + $args2 \\ + --reference $fasta \\ + --threads $task.cpus \\ + -R $bed \\ + -o $prefix + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + hap.py: $VERSION_HAP + pre.py: $VERSION_PRE + END_VERSIONS + """ +} diff --git a/modules/happy/meta.yml b/modules/happy/meta.yml new file mode 100644 index 00000000..52d98e04 --- /dev/null +++ b/modules/happy/meta.yml @@ -0,0 +1,51 @@ +name: "happy" +description: Hap.py is a tool to compare diploid genotypes at haplotype level. Rather than comparing VCF records row by row, hap.py will generate and match alternate sequences in a superlocus. A superlocus is a small region of the genome (sized between 1 and around 1000 bp) that contains one or more variants. +keywords: + - happy + - benchmark + - haplotype +tools: + - "happy": + description: "Haplotype VCF comparison tools" + homepage: "https://www.illumina.com/products/by-type/informatics-products/basespace-sequence-hub/apps/hap-py-benchmarking.html" + documentation: "https://github.com/Illumina/hap.py" + tool_dev_url: "https://github.com/Illumina/hap.py" + doi: "" + licence: "['BSD-2-clause']" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - truth_vcf: + type: file + description: gold standard VCF file + pattern: "*.{vcf,vcf.gz}" + - query_vcf: + type: file + description: VCF file to query + pattern: "*.{vcf,vcf.gz}" + - bed: + type: file + description: BED file + pattern: "*.bed" + - fasta: + type: file + description: FASTA file of the reference genome + pattern: "*.{fa,fasta}" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@nvnieuwk" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index f4feb844..c14ffc8e 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -891,6 +891,10 @@ hamronization/summarize: - modules/hamronization/summarize/** - tests/modules/hamronization/summarize/** +happy: + - modules/happy/** + - tests/modules/happy/** + hicap: - modules/hicap/** - tests/modules/hicap/** diff --git a/tests/modules/happy/main.nf b/tests/modules/happy/main.nf new file mode 100644 index 00000000..0ce23b50 --- /dev/null +++ b/tests/modules/happy/main.nf @@ -0,0 +1,39 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { HAPPY } from '../../../modules/happy/main.nf' + +workflow test_happy_vcf { + + input = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_rnaseq_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_genome21_indels_vcf_gz'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + ] + + fasta = Channel.value([ + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ]) + + HAPPY ( input, fasta ) +} + +workflow test_happy_gvcf { + + input = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_rnaseq_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['illumina']['test_genome_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + ] + + fasta = Channel.value([ + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ]) + + HAPPY ( input, fasta ) +} diff --git a/tests/modules/happy/nextflow.config b/tests/modules/happy/nextflow.config new file mode 100644 index 00000000..50f50a7a --- /dev/null +++ b/tests/modules/happy/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml new file mode 100644 index 00000000..d85d42a2 --- /dev/null +++ b/tests/modules/happy/test.yml @@ -0,0 +1,7 @@ +- name: happy test_happy + command: nextflow run tests/modules/happy -entry test_happy -c tests/config/nextflow.config + tags: + - happy + files: + - path: output/happy/versions.yml + md5sum: ab8944c1b24e55bab311a9d389c75c90 From 37338ecee30a7210e9a40d993370fd25bddf003d Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:00:02 +0200 Subject: [PATCH 81/91] Finished the module --- modules/happy/meta.yml | 12 ++++++++++++ tests/modules/happy/test.yml | 26 +++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/modules/happy/meta.yml b/modules/happy/meta.yml index 52d98e04..28142d9b 100644 --- a/modules/happy/meta.yml +++ b/modules/happy/meta.yml @@ -42,6 +42,18 @@ output: description: | Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - summary: + type: file + description: A CSV file containing the summary of the benchmarking + pattern: "*.summary.csv" + - extended: + type: file + description: A CSV file containing extended info of the benchmarking + pattern: "*.extended.csv" + - runinfo: + type: file + description: A JSON file containing the run info + pattern: "*.runinfo.json" - versions: type: file description: File containing software versions diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index d85d42a2..7dddca17 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -1,7 +1,27 @@ -- name: happy test_happy - command: nextflow run tests/modules/happy -entry test_happy -c tests/config/nextflow.config +- name: happy test_happy_vcf + command: nextflow run tests/modules/happy -entry test_happy_vcf -c tests/config/nextflow.config tags: - happy files: + - path: output/happy/test.extended.csv + md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 + - path: output/happy/test.runinfo.json + contains: '"name": "hap.py"' + - path: output/happy/test.summary.csv + md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml - md5sum: ab8944c1b24e55bab311a9d389c75c90 + md5sum: 2c2b8249f9f52194da7f09076dbb5019 + +- name: happy test_happy_gvcf + command: nextflow run tests/modules/happy -entry test_happy_gvcf -c tests/config/nextflow.config + tags: + - happy + files: + - path: output/happy/test.extended.csv + md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 + - path: output/happy/test.runinfo.json + contains: '"name": "hap.py"' + - path: output/happy/test.summary.csv + md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 + - path: output/happy/versions.yml + md5sum: 75331161af9ec046d045ffcba83abddf From 065a5d6a7f13a50ad7965fdc770df50363bde3d7 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:05:21 +0200 Subject: [PATCH 82/91] Fixed a linting issue --- modules/happy/main.nf | 2 +- modules/happy/meta.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/happy/main.nf b/modules/happy/main.nf index 68059dd8..1338686e 100644 --- a/modules/happy/main.nf +++ b/modules/happy/main.nf @@ -25,7 +25,7 @@ process HAPPY { def args = task.ext.args ?: '' def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - + """ pre.py \\ $args \\ diff --git a/modules/happy/meta.yml b/modules/happy/meta.yml index 28142d9b..aba9410b 100644 --- a/modules/happy/meta.yml +++ b/modules/happy/meta.yml @@ -45,7 +45,7 @@ output: - summary: type: file description: A CSV file containing the summary of the benchmarking - pattern: "*.summary.csv" + pattern: "*.summary.csv" - extended: type: file description: A CSV file containing extended info of the benchmarking @@ -60,4 +60,4 @@ output: pattern: "versions.yml" authors: - - "@nvnieuwk" + - "@nvnieuwk" \ No newline at end of file From 177a7e49f249952d8b3271c454529a768f16af43 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:06:42 +0200 Subject: [PATCH 83/91] Fixed a linting issue --- modules/happy/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/happy/meta.yml b/modules/happy/meta.yml index aba9410b..d732df22 100644 --- a/modules/happy/meta.yml +++ b/modules/happy/meta.yml @@ -60,4 +60,4 @@ output: pattern: "versions.yml" authors: - - "@nvnieuwk" \ No newline at end of file + - "@nvnieuwk" From c1393bf999e4e967d24d7ec3d2e864c5d0a856e3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:14:01 +0200 Subject: [PATCH 84/91] Possible fix for the pytest errors? --- tests/modules/happy/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index 7dddca17..f5bdef52 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -6,7 +6,7 @@ - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 - path: output/happy/test.runinfo.json - contains: '"name": "hap.py"' + contains: 'hap.py' - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml @@ -20,7 +20,7 @@ - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 - path: output/happy/test.runinfo.json - contains: '"name": "hap.py"' + contains: 'hap.py' - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml From 505ab038037df6a7d7af1db1cae3354b8f713364 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:14:53 +0200 Subject: [PATCH 85/91] fixed a description --- modules/happy/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/happy/meta.yml b/modules/happy/meta.yml index d732df22..6b5b86cf 100644 --- a/modules/happy/meta.yml +++ b/modules/happy/meta.yml @@ -25,7 +25,7 @@ input: pattern: "*.{vcf,vcf.gz}" - query_vcf: type: file - description: VCF file to query + description: VCF/GVCF file to query pattern: "*.{vcf,vcf.gz}" - bed: type: file From 17dbd2bb82093cd6ca1d1439c5c7815eb0b9a2a6 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:18:56 +0200 Subject: [PATCH 86/91] Linting issue fixed --- tests/modules/happy/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index f5bdef52..937df32e 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -6,7 +6,7 @@ - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 - path: output/happy/test.runinfo.json - contains: 'hap.py' + contains: hap.py - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml @@ -20,7 +20,7 @@ - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 - path: output/happy/test.runinfo.json - contains: 'hap.py' + contains: hap.py - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml From 583ea138006d8c49d4821bbfaeba81408ff57053 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:22:55 +0200 Subject: [PATCH 87/91] Another try to fix the pytest issues --- tests/modules/happy/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index 937df32e..c0a34387 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -6,7 +6,7 @@ - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 - path: output/happy/test.runinfo.json - contains: hap.py + contains: dist - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml @@ -20,7 +20,7 @@ - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 - path: output/happy/test.runinfo.json - contains: hap.py + contains: dist - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml From 5889d6c9233fae612dbce8d333773f1a8b1d47e9 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:46:15 +0200 Subject: [PATCH 88/91] Another possible fix for the pytest issues --- tests/modules/happy/test.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index c0a34387..9af6cc83 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -6,7 +6,9 @@ - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 - path: output/happy/test.runinfo.json - contains: dist + contains: + - 'dist' + - 'python_version' - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml @@ -20,7 +22,9 @@ - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 - path: output/happy/test.runinfo.json - contains: dist + contains: + - 'dist' + - 'python_version' - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml From 3c661386728c3552da55db5252ef2bae8f113220 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 14:49:17 +0200 Subject: [PATCH 89/91] Linting issue + removed the contains lines --- tests/modules/happy/test.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/test.yml index 9af6cc83..0fd26e5b 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/test.yml @@ -6,9 +6,6 @@ - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 - path: output/happy/test.runinfo.json - contains: - - 'dist' - - 'python_version' - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml @@ -22,9 +19,6 @@ - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 - path: output/happy/test.runinfo.json - contains: - - 'dist' - - 'python_version' - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml From f875aec46b9ab231ab96dfa4bb3b62c53fb821ee Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 15:29:48 +0200 Subject: [PATCH 90/91] Split the module into two parts --- modules/happy/{ => happy}/main.nf | 21 ++----- modules/happy/{ => happy}/meta.yml | 6 +- modules/happy/prepy/main.nf | 41 ++++++++++++++ modules/happy/prepy/meta.yml | 55 +++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/happy/{ => happy}/main.nf | 6 +- .../modules/happy/{ => happy}/nextflow.config | 0 tests/modules/happy/{ => happy}/test.yml | 14 +++-- tests/modules/happy/prepy/main.nf | 37 +++++++++++++ tests/modules/happy/prepy/nextflow.config | 5 ++ tests/modules/happy/prepy/test.yml | 19 +++++++ 11 files changed, 182 insertions(+), 26 deletions(-) rename modules/happy/{ => happy}/main.nf (74%) rename modules/happy/{ => happy}/meta.yml (93%) create mode 100644 modules/happy/prepy/main.nf create mode 100644 modules/happy/prepy/meta.yml rename tests/modules/happy/{ => happy}/main.nf (90%) rename tests/modules/happy/{ => happy}/nextflow.config (100%) rename tests/modules/happy/{ => happy}/test.yml (58%) create mode 100644 tests/modules/happy/prepy/main.nf create mode 100644 tests/modules/happy/prepy/nextflow.config create mode 100644 tests/modules/happy/prepy/test.yml diff --git a/modules/happy/main.nf b/modules/happy/happy/main.nf similarity index 74% rename from modules/happy/main.nf rename to modules/happy/happy/main.nf index 1338686e..1bb99117 100644 --- a/modules/happy/main.nf +++ b/modules/happy/happy/main.nf @@ -1,7 +1,6 @@ -def VERSION_HAP = '0.3.14' -def VERSION_PRE = '0.3.14' +def VERSION = '0.3.14' -process HAPPY { +process HAPPY_HAPPY { tag "$meta.id" label 'process_medium' @@ -23,22 +22,13 @@ process HAPPY { script: def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" """ - pre.py \\ - $args \\ - -R $bed \\ - --reference $fasta \\ - --threads $task.cpus \\ - $query_vcf \\ - ${prefix}_preprocessed.vcf.gz - hap.py \\ $truth_vcf \\ - ${prefix}_preprocessed.vcf.gz \\ - $args2 \\ + $query_vcf \\ + $args \\ --reference $fasta \\ --threads $task.cpus \\ -R $bed \\ @@ -46,8 +36,7 @@ process HAPPY { cat <<-END_VERSIONS > versions.yml "${task.process}": - hap.py: $VERSION_HAP - pre.py: $VERSION_PRE + hap.py: $VERSION END_VERSIONS """ } diff --git a/modules/happy/meta.yml b/modules/happy/happy/meta.yml similarity index 93% rename from modules/happy/meta.yml rename to modules/happy/happy/meta.yml index 6b5b86cf..8ec762d5 100644 --- a/modules/happy/meta.yml +++ b/modules/happy/happy/meta.yml @@ -1,4 +1,4 @@ -name: "happy" +name: "happy_happy" description: Hap.py is a tool to compare diploid genotypes at haplotype level. Rather than comparing VCF records row by row, hap.py will generate and match alternate sequences in a superlocus. A superlocus is a small region of the genome (sized between 1 and around 1000 bp) that contains one or more variants. keywords: - happy @@ -35,6 +35,10 @@ input: type: file description: FASTA file of the reference genome pattern: "*.{fa,fasta}" + - fasta_fai: + type: file + description: The index of the reference FASTA + pattern: "*.fai" output: - meta: diff --git a/modules/happy/prepy/main.nf b/modules/happy/prepy/main.nf new file mode 100644 index 00000000..936f56ea --- /dev/null +++ b/modules/happy/prepy/main.nf @@ -0,0 +1,41 @@ +def VERSION = '0.3.14' + +process HAPPY_PREPY { + tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? "bioconda::hap.py=0.3.14" : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/hap.py:0.3.14--py27h5c5a3ab_0': + 'quay.io/biocontainers/hap.py:0.3.14--py27h5c5a3ab_0' }" + + input: + tuple val(meta), path(vcf), path(bed) + tuple path(fasta), path(fasta_fai) + + output: + tuple val(meta), path('*.vcf.gz') , emit: preprocessed_vcf + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + + """ + pre.py \\ + $args \\ + -R $bed \\ + --reference $fasta \\ + --threads $task.cpus \\ + $vcf \\ + ${prefix}.vcf.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + pre.py: $VERSION + END_VERSIONS + """ +} diff --git a/modules/happy/prepy/meta.yml b/modules/happy/prepy/meta.yml new file mode 100644 index 00000000..7e27a652 --- /dev/null +++ b/modules/happy/prepy/meta.yml @@ -0,0 +1,55 @@ +name: "happy_prepy" +description: Pre.py is a preprocessing tool made to preprocess VCF files for Hap.py +keywords: + - happy + - benchmark + - haplotype +tools: + - "happy": + description: "Haplotype VCF comparison tools" + homepage: "https://www.illumina.com/products/by-type/informatics-products/basespace-sequence-hub/apps/hap-py-benchmarking.html" + documentation: "https://github.com/Illumina/hap.py" + tool_dev_url: "https://github.com/Illumina/hap.py" + doi: "" + licence: "['BSD-2-clause']" + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: VCF file to preprocess + pattern: "*.{vcf,vcf.gz}" + - bed: + type: file + description: BED file + pattern: "*.bed" + - fasta: + type: file + description: FASTA file of the reference genome + pattern: "*.{fa,fasta}" + - fasta_fai: + type: file + description: The index of the reference FASTA + pattern: "*.fai" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - vcf: + type: file + description: A preprocessed VCF file + pattern: "*.vcf.gz" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + +authors: + - "@nvnieuwk" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index c14ffc8e..4f85ba7d 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -895,6 +895,10 @@ happy: - modules/happy/** - tests/modules/happy/** +happy/prepy: + - modules/happy/prepy/** + - tests/modules/happy/prepy/** + hicap: - modules/hicap/** - tests/modules/hicap/** diff --git a/tests/modules/happy/main.nf b/tests/modules/happy/happy/main.nf similarity index 90% rename from tests/modules/happy/main.nf rename to tests/modules/happy/happy/main.nf index 0ce23b50..515a657b 100644 --- a/tests/modules/happy/main.nf +++ b/tests/modules/happy/happy/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { HAPPY } from '../../../modules/happy/main.nf' +include { HAPPY_HAPPY } from '../../../../modules/happy/happy/main.nf' workflow test_happy_vcf { @@ -18,7 +18,7 @@ workflow test_happy_vcf { file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) ]) - HAPPY ( input, fasta ) + HAPPY_HAPPY ( input, fasta ) } workflow test_happy_gvcf { @@ -35,5 +35,5 @@ workflow test_happy_gvcf { file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) ]) - HAPPY ( input, fasta ) + HAPPY_HAPPY ( input, fasta ) } diff --git a/tests/modules/happy/nextflow.config b/tests/modules/happy/happy/nextflow.config similarity index 100% rename from tests/modules/happy/nextflow.config rename to tests/modules/happy/happy/nextflow.config diff --git a/tests/modules/happy/test.yml b/tests/modules/happy/happy/test.yml similarity index 58% rename from tests/modules/happy/test.yml rename to tests/modules/happy/happy/test.yml index 0fd26e5b..89a1f012 100644 --- a/tests/modules/happy/test.yml +++ b/tests/modules/happy/happy/test.yml @@ -1,7 +1,8 @@ -- name: happy test_happy_vcf - command: nextflow run tests/modules/happy -entry test_happy_vcf -c tests/config/nextflow.config +- name: happy happy test_happy_vcf + command: nextflow run tests/modules/happy/happy -entry test_happy_vcf -c tests/config/nextflow.config tags: - happy + - happy/happy files: - path: output/happy/test.extended.csv md5sum: ef79c7c789ef4f146ca2e50dafaf22b3 @@ -9,12 +10,13 @@ - path: output/happy/test.summary.csv md5sum: f8aa5d36d3c48dede2f607fd565894ad - path: output/happy/versions.yml - md5sum: 2c2b8249f9f52194da7f09076dbb5019 + md5sum: 82243bf6dbdc71aa63211ee2a89f47f2 -- name: happy test_happy_gvcf - command: nextflow run tests/modules/happy -entry test_happy_gvcf -c tests/config/nextflow.config +- name: happy happy test_happy_gvcf + command: nextflow run tests/modules/happy/happy -entry test_happy_gvcf -c tests/config/nextflow.config tags: - happy + - happy/happy files: - path: output/happy/test.extended.csv md5sum: 3d5c21b67a259a3f6dcb088d55b86cd3 @@ -22,4 +24,4 @@ - path: output/happy/test.summary.csv md5sum: 03044e9bb5a0c6f0947b7e910fc8a558 - path: output/happy/versions.yml - md5sum: 75331161af9ec046d045ffcba83abddf + md5sum: 551fa216952d6f5de78e6e453b92aaab diff --git a/tests/modules/happy/prepy/main.nf b/tests/modules/happy/prepy/main.nf new file mode 100644 index 00000000..0b511104 --- /dev/null +++ b/tests/modules/happy/prepy/main.nf @@ -0,0 +1,37 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { HAPPY_PREPY } from '../../../../modules/happy/prepy/main.nf' + +workflow test_happy_prepy_vcf { + + input = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_genome21_indels_vcf_gz'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + ] + + fasta = Channel.value([ + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ]) + + HAPPY_PREPY ( input, fasta ) +} + +workflow test_happy_prepy_gvcf { + + input = [ + [ id:'test' ], // meta map + file(params.test_data['homo_sapiens']['illumina']['test_genome_vcf'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) + ] + + fasta = Channel.value([ + file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) + ]) + + HAPPY_PREPY ( input, fasta ) +} diff --git a/tests/modules/happy/prepy/nextflow.config b/tests/modules/happy/prepy/nextflow.config new file mode 100644 index 00000000..50f50a7a --- /dev/null +++ b/tests/modules/happy/prepy/nextflow.config @@ -0,0 +1,5 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + +} \ No newline at end of file diff --git a/tests/modules/happy/prepy/test.yml b/tests/modules/happy/prepy/test.yml new file mode 100644 index 00000000..a0a45c38 --- /dev/null +++ b/tests/modules/happy/prepy/test.yml @@ -0,0 +1,19 @@ +- name: happy prepy test_happy_prepy_vcf + command: nextflow run tests/modules/happy/prepy -entry test_happy_prepy_vcf -c tests/config/nextflow.config + tags: + - happy/prepy + - happy + files: + - path: output/happy/test.vcf.gz + - path: output/happy/versions.yml + md5sum: 814d20f1f29f23a3d21012748a5d6393 + +- name: happy prepy test_happy_prepy_gvcf + command: nextflow run tests/modules/happy/prepy -entry test_happy_prepy_gvcf -c tests/config/nextflow.config + tags: + - happy/prepy + - happy + files: + - path: output/happy/test.vcf.gz + - path: output/happy/versions.yml + md5sum: 970a54de46e68ef6d5228a26eaa4c8e7 From 42bdc5471e1591ba6c8272dcd8a955ea21fec2e3 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Wed, 4 May 2022 15:40:34 +0200 Subject: [PATCH 91/91] Changed the pytest location of happy --- tests/config/pytest_modules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 4f85ba7d..32a28477 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -891,9 +891,9 @@ hamronization/summarize: - modules/hamronization/summarize/** - tests/modules/hamronization/summarize/** -happy: - - modules/happy/** - - tests/modules/happy/** +happy/happy: + - modules/happy/happy/** + - tests/modules/happy/happy/** happy/prepy: - modules/happy/prepy/**