From 38b450d0be865af4f60dd4674a4fc47ddbf1e6e1 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 09:54:29 +0000 Subject: [PATCH 01/12] hifiasm copied from fastqc --- software/hifiasm/functions.nf | 59 ++++++++++++++++++++++++++++ software/hifiasm/main.nf | 47 +++++++++++++++++++++++ software/hifiasm/meta.yml | 72 +++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 software/hifiasm/functions.nf create mode 100644 software/hifiasm/main.nf create mode 100644 software/hifiasm/meta.yml diff --git a/software/hifiasm/functions.nf b/software/hifiasm/functions.nf new file mode 100644 index 00000000..d25eea86 --- /dev/null +++ b/software/hifiasm/functions.nf @@ -0,0 +1,59 @@ +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf new file mode 100644 index 00000000..4e847876 --- /dev/null +++ b/software/hifiasm/main.nf @@ -0,0 +1,47 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +options = initOptions(params.options) + +process FASTQC { + tag "$meta.id" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + + conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + } else { + container "quay.io/biocontainers/fastqc:0.11.9--0" + } + + input: + tuple val(meta), path(reads) + + output: + tuple val(meta), path("*.html"), emit: html + tuple val(meta), path("*.zip") , emit: zip + path "*.version.txt" , emit: version + + script: + // Add soft-links to original FastQs for consistent naming in pipeline + def software = getSoftwareName(task.process) + def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" + if (meta.single_end) { + """ + [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz + fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + """ + } else { + """ + [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz + [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz + fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + """ + } +} diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml new file mode 100644 index 00000000..413aad8d --- /dev/null +++ b/software/hifiasm/meta.yml @@ -0,0 +1,72 @@ +name: fastqc +description: Run FastQC on sequenced reads +keywords: + - quality control + - qc + - adapters + - fastq +tools: + - fastqc: + description: | + FastQC gives general quality metrics about your reads. + It provides information about the quality score distribution + across your reads, the per base sequence content (%A/C/G/T). + You get information about adapter contamination and other + overrepresented sequences. + homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ + documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - html: + type: file + description: FastQC report + pattern: "*_{fastqc.html}" + - zip: + type: file + description: FastQC report archive + pattern: "*_{fastqc.zip}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@drpatelh" + - "@grst" + - "@ewels" + - "@FelixKrueger" From 2f675fbe7c60a67649dd4a02e72e2f1ee79f24d3 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 11:37:06 +0000 Subject: [PATCH 02/12] hifiasm tests init from fastqc --- tests/software/hifiasm/main.nf | 28 ++++++++++++++++++++++++++++ tests/software/hifiasm/test.yml | 19 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tests/software/hifiasm/main.nf create mode 100644 tests/software/hifiasm/test.yml diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf new file mode 100644 index 00000000..f10b2357 --- /dev/null +++ b/tests/software/hifiasm/main.nf @@ -0,0 +1,28 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { FASTQC } from '../../../software/fastqc/main.nf' addParams( options: [:] ) + +/* + * Test with single-end data + */ +workflow test_fastqc_single_end { + + def input = [] + input = [ [ id:'test', single_end:true ], // meta map + [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] + FASTQC ( input ) +} + +/* + * Test with paired-end data + */ +workflow test_fastqc_paired_end { + + def input = [] + input = [[id: 'test', single_end: false], // meta map + [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), + file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] + FASTQC (input) +} diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml new file mode 100644 index 00000000..0fe0ebe0 --- /dev/null +++ b/tests/software/hifiasm/test.yml @@ -0,0 +1,19 @@ +- name: fastqc single-end + command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_single_end -c tests/config/nextflow.config + tags: + - fastqc + - fastqc_single_end + files: + - path: ./output/fastqc/test_fastqc.html + - path: ./output/fastqc/test_fastqc.zip + +- name: fastqc paired-end + command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_paired_end -c tests/config/nextflow.config + tags: + - fastqc + - fastqc_paired_end + files: + - path: ./output/fastqc/test_1_fastqc.html + - path: ./output/fastqc/test_2_fastqc.html + - path: ./output/fastqc/test_2_fastqc.zip + - path: ./output/fastqc/test_1_fastqc.zip From 390c835f9fc47574654aa0bfd55556bf376e98f2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 11:54:42 +0000 Subject: [PATCH 03/12] meta.yml init; test.yml and main.nf for printing version --- software/hifiasm/meta.yml | 23 ++++++++++------------- tests/software/hifiasm/main.nf | 18 +++++++++--------- tests/software/hifiasm/test.yml | 22 +++++----------------- 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml index 413aad8d..9ef62628 100644 --- a/software/hifiasm/meta.yml +++ b/software/hifiasm/meta.yml @@ -1,20 +1,17 @@ -name: fastqc -description: Run FastQC on sequenced reads +name: hifiasm +description: Run hifiasm on Pacbio HiFi reads keywords: - - quality control - - qc - - adapters + - genome assembly + - pacbaio + - hifi + - long reads - fastq tools: - - fastqc: + - hifiasm: description: | - FastQC gives general quality metrics about your reads. - It provides information about the quality score distribution - across your reads, the per base sequence content (%A/C/G/T). - You get information about adapter contamination and other - overrepresented sequences. - homepage: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/ - documentation: https://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/ + hifiasm generates a genome assembly from long Pacbio HiFi reads. ...Haplotype resolution ...Using short reads. + homepage: https://github.com/chhylp123/hifiasm + documentation: https://github.com/chhylp123/hifiasm params: - outdir: type: string diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf index f10b2357..66824584 100644 --- a/tests/software/hifiasm/main.nf +++ b/tests/software/hifiasm/main.nf @@ -2,27 +2,27 @@ nextflow.enable.dsl = 2 -include { FASTQC } from '../../../software/fastqc/main.nf' addParams( options: [:] ) +include { HIFIASM } from '../../../software/hifiasm/main.nf' addParams( options: [:] ) -/* +/* * Test with single-end data */ -workflow test_fastqc_single_end { +/* workflow test_fastqc_single_end { def input = [] input = [ [ id:'test', single_end:true ], // meta map [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] FASTQC ( input ) -} +} */ /* - * Test with paired-end data + * Test version printing */ -workflow test_fastqc_paired_end { +workflow test_hifiasm_version { - def input = [] +/* def input = [] input = [[id: 'test', single_end: false], // meta map [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), - file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] - FASTQC (input) + file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] */ + HIFIASM () } diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml index 0fe0ebe0..3825b238 100644 --- a/tests/software/hifiasm/test.yml +++ b/tests/software/hifiasm/test.yml @@ -1,19 +1,7 @@ -- name: fastqc single-end - command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_single_end -c tests/config/nextflow.config +- name: hifiasm + command: nextflow run ./tests/software/hifiasm/ -entry test_hifiasm_version -c tests/config/nextflow.config tags: - - fastqc - - fastqc_single_end + - hifiasm + - hifiasm_version files: - - path: ./output/fastqc/test_fastqc.html - - path: ./output/fastqc/test_fastqc.zip - -- name: fastqc paired-end - command: nextflow run ./tests/software/fastqc/ -entry test_fastqc_paired_end -c tests/config/nextflow.config - tags: - - fastqc - - fastqc_paired_end - files: - - path: ./output/fastqc/test_1_fastqc.html - - path: ./output/fastqc/test_2_fastqc.html - - path: ./output/fastqc/test_2_fastqc.zip - - path: ./output/fastqc/test_1_fastqc.zip + - path: ./output/hifiasm/VERSION.txt From b58233794b05b287d0887c6803c6c2f7abf10db4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 13:14:43 +0000 Subject: [PATCH 04/12] Add hifiasm version printing --- software/hifiasm/main.nf | 33 +++++++++++++++++++-------------- tests/software/hifiasm/test.yml | 4 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf index 4e847876..7a364be2 100644 --- a/software/hifiasm/main.nf +++ b/software/hifiasm/main.nf @@ -4,33 +4,34 @@ include { initOptions; saveFiles; getSoftwareName } from './functions' params.options = [:] options = initOptions(params.options) -process FASTQC { - tag "$meta.id" - label 'process_medium' +process HIFIASM { + //tag "$meta.id" + label 'process_high' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:meta.id) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:'') } // publish_id:meta.id) } - conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) + conda (params.enable_conda ? "bioconda::hifiasm=0.14" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" + container "https://depot.galaxyproject.org/singularity/hifiasm:0.14--h8b12597_0" } else { - container "quay.io/biocontainers/fastqc:0.11.9--0" + container "quay.io/biocontainers/hifiasm:0.14--h8b12597_0" } - input: - tuple val(meta), path(reads) +/* input: + tuple val(meta), path(reads) */ output: - tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip +/* tuple val(meta), path("*.html"), emit: html + tuple val(meta), path("*.zip") , emit: zip */ path "*.version.txt" , emit: version script: // Add soft-links to original FastQs for consistent naming in pipeline def software = getSoftwareName(task.process) - def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - if (meta.single_end) { + //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" + +/* if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz @@ -43,5 +44,9 @@ process FASTQC { fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt """ - } + } */ + + """ + hifiasm --version > ${software}.version.txt || exit 0 + """ } diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml index 3825b238..e639f099 100644 --- a/tests/software/hifiasm/test.yml +++ b/tests/software/hifiasm/test.yml @@ -1,7 +1,7 @@ - name: hifiasm - command: nextflow run ./tests/software/hifiasm/ -entry test_hifiasm_version -c tests/config/nextflow.config + command: nextflow run ./tests/software/hifiasm -entry test_hifiasm_version -c tests/config/nextflow.config tags: - hifiasm - hifiasm_version files: - - path: ./output/hifiasm/VERSION.txt + - path: ./output/hifiasm/hifiasm.version.txt From 5a270c5ba7a0b382d2ca48ce42f7399a6003fa7d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Tue, 16 Mar 2021 15:55:59 +0000 Subject: [PATCH 05/12] Removed spaced on an empty line --- software/hifiasm/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf index 7a364be2..8686b6bc 100644 --- a/software/hifiasm/main.nf +++ b/software/hifiasm/main.nf @@ -30,7 +30,7 @@ process HIFIASM { // Add soft-links to original FastQs for consistent naming in pipeline def software = getSoftwareName(task.process) //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - + /* if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz From 7aaf0b72486a6948f8e93f5905e56339dbca9ee5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sidorov Date: Sun, 2 May 2021 14:09:03 +0100 Subject: [PATCH 06/12] Reverted hifiasm from main --- software/hifiasm/functions.nf | 59 ---------------------------- software/hifiasm/main.nf | 52 ------------------------- software/hifiasm/meta.yml | 69 --------------------------------- tests/software/hifiasm/main.nf | 28 ------------- tests/software/hifiasm/test.yml | 7 ---- 5 files changed, 215 deletions(-) delete mode 100644 software/hifiasm/functions.nf delete mode 100644 software/hifiasm/main.nf delete mode 100644 software/hifiasm/meta.yml delete mode 100644 tests/software/hifiasm/main.nf delete mode 100644 tests/software/hifiasm/test.yml diff --git a/software/hifiasm/functions.nf b/software/hifiasm/functions.nf deleted file mode 100644 index d25eea86..00000000 --- a/software/hifiasm/functions.nf +++ /dev/null @@ -1,59 +0,0 @@ -/* - * ----------------------------------------------------- - * Utility functions used in nf-core DSL2 module files - * ----------------------------------------------------- - */ - -/* - * Extract name of software tool from process name using $task.process - */ -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - -/* - * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules - */ -def initOptions(Map args) { - def Map options = [:] - options.args = args.args ?: '' - options.args2 = args.args2 ?: '' - options.publish_by_id = args.publish_by_id ?: false - options.publish_dir = args.publish_dir ?: '' - options.publish_files = args.publish_files - options.suffix = args.suffix ?: '' - return options -} - -/* - * Tidy up and join elements of a list to return a path string - */ -def getPathFromList(path_list) { - def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries - paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes - return paths.join('/') -} - -/* - * Function to save/publish module results - */ -def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_id) { - path_list.add(args.publish_id) - } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } - } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" - } - } -} diff --git a/software/hifiasm/main.nf b/software/hifiasm/main.nf deleted file mode 100644 index 8686b6bc..00000000 --- a/software/hifiasm/main.nf +++ /dev/null @@ -1,52 +0,0 @@ -// Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' - -params.options = [:] -options = initOptions(params.options) - -process HIFIASM { - //tag "$meta.id" - label 'process_high' - publishDir "${params.outdir}", - mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:'') } // publish_id:meta.id) } - - conda (params.enable_conda ? "bioconda::hifiasm=0.14" : null) - if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/hifiasm:0.14--h8b12597_0" - } else { - container "quay.io/biocontainers/hifiasm:0.14--h8b12597_0" - } - -/* input: - tuple val(meta), path(reads) */ - - output: -/* tuple val(meta), path("*.html"), emit: html - tuple val(meta), path("*.zip") , emit: zip */ - path "*.version.txt" , emit: version - - script: - // Add soft-links to original FastQs for consistent naming in pipeline - def software = getSoftwareName(task.process) - //def prefix = options.suffix ? "${meta.id}.${options.suffix}" : "${meta.id}" - -/* if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt - """ - } */ - - """ - hifiasm --version > ${software}.version.txt || exit 0 - """ -} diff --git a/software/hifiasm/meta.yml b/software/hifiasm/meta.yml deleted file mode 100644 index 9ef62628..00000000 --- a/software/hifiasm/meta.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: hifiasm -description: Run hifiasm on Pacbio HiFi reads -keywords: - - genome assembly - - pacbaio - - hifi - - long reads - - fastq -tools: - - hifiasm: - description: | - hifiasm generates a genome assembly from long Pacbio HiFi reads. ...Haplotype resolution ...Using short reads. - homepage: https://github.com/chhylp123/hifiasm - documentation: https://github.com/chhylp123/hifiasm -params: - - outdir: - type: string - description: | - The pipeline's output directory. By default, the module will - output files into `$params.outdir/` - - publish_dir_mode: - type: string - description: | - Value for the Nextflow `publishDir` mode parameter. - Available: symlink, rellink, link, copy, copyNoFollow, move. - - enable_conda: - type: boolean - description: | - Run the module with Conda using the software specified - via the `conda` directive - - singularity_pull_docker_container: - type: boolean - description: | - Instead of directly downloading Singularity images for use with Singularity, - force the workflow to pull and convert Docker containers instead. -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - html: - type: file - description: FastQC report - pattern: "*_{fastqc.html}" - - zip: - type: file - description: FastQC report archive - pattern: "*_{fastqc.zip}" - - version: - type: file - description: File containing software version - pattern: "*.{version.txt}" -authors: - - "@drpatelh" - - "@grst" - - "@ewels" - - "@FelixKrueger" diff --git a/tests/software/hifiasm/main.nf b/tests/software/hifiasm/main.nf deleted file mode 100644 index 66824584..00000000 --- a/tests/software/hifiasm/main.nf +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { HIFIASM } from '../../../software/hifiasm/main.nf' addParams( options: [:] ) - -/* - * Test with single-end data - */ -/* workflow test_fastqc_single_end { - - def input = [] - input = [ [ id:'test', single_end:true ], // meta map - [ file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true) ] ] - FASTQC ( input ) -} */ - -/* - * Test version printing - */ -workflow test_hifiasm_version { - -/* def input = [] - input = [[id: 'test', single_end: false], // meta map - [file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_1.fastq.gz", checkIfExists: true), - file("${launchDir}/tests/data/genomics/sarscov2/fastq/test_2.fastq.gz", checkIfExists: true)]] */ - HIFIASM () -} diff --git a/tests/software/hifiasm/test.yml b/tests/software/hifiasm/test.yml deleted file mode 100644 index e639f099..00000000 --- a/tests/software/hifiasm/test.yml +++ /dev/null @@ -1,7 +0,0 @@ -- name: hifiasm - command: nextflow run ./tests/software/hifiasm -entry test_hifiasm_version -c tests/config/nextflow.config - tags: - - hifiasm - - hifiasm_version - files: - - path: ./output/hifiasm/hifiasm.version.txt From 1abcc938379595db0840dc32d1681cdebc268f74 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Fri, 28 Jan 2022 09:14:47 +0000 Subject: [PATCH 07/12] Ignores version file checking if testing for error --- tests/test_versions_yml.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index 2f78ab2e..19ba1cf9 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -16,7 +16,8 @@ def _get_workflow_names(): # test_config = yaml.safe_load(f.read_text()) test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) for workflow in test_config: - yield workflow["name"] + if 'exit_code' not in workflow: + yield workflow["name"] @pytest.mark.workflow(*_get_workflow_names()) From 70ea9ac9a863061e22e7b5a8dd3fe07a29ecf4c4 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Fri, 28 Jan 2022 09:27:28 +0000 Subject: [PATCH 08/12] Added comment --- tests/test_versions_yml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_versions_yml.py b/tests/test_versions_yml.py index 19ba1cf9..5d0bb39e 100644 --- a/tests/test_versions_yml.py +++ b/tests/test_versions_yml.py @@ -16,7 +16,9 @@ def _get_workflow_names(): # test_config = yaml.safe_load(f.read_text()) test_config = yaml.load(f.read_text(), Loader=yaml.BaseLoader) for workflow in test_config: - if 'exit_code' not in workflow: + # https://github.com/nf-core/modules/pull/1242 - added to cover tests + # that expect an error and therefore will not generate a versions.yml + if 'exit_code' not in workflow: yield workflow["name"] From 768073be6390c620e133230dea376466383832f5 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 11:46:19 +0100 Subject: [PATCH 09/12] bowtie2|Added support for large indexes --- modules/bowtie2/align/main.nf | 4 ++ tests/modules/bowtie2/align/main.nf | 2 +- tests/modules/bowtie2/align/nextflow.config | 13 ++++++- tests/modules/bowtie2/align/test.yml | 43 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) diff --git a/modules/bowtie2/align/main.nf b/modules/bowtie2/align/main.nf index 20b08f72..e68dbf24 100644 --- a/modules/bowtie2/align/main.nf +++ b/modules/bowtie2/align/main.nf @@ -26,6 +26,8 @@ process BOWTIE2_ALIGN { def unaligned = save_unaligned ? "--un-gz ${prefix}.unmapped.fastq.gz" : '' """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` + [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` + [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 bowtie2 \\ -x \$INDEX \\ -U $reads \\ @@ -46,6 +48,8 @@ process BOWTIE2_ALIGN { def unaligned = save_unaligned ? "--un-conc-gz ${prefix}.unmapped.fastq.gz" : '' """ INDEX=`find -L ./ -name "*.rev.1.bt2" | sed 's/.rev.1.bt2//'` + [ -z "\$INDEX" ] && INDEX=`find -L ./ -name "*.rev.1.bt2l" | sed 's/.rev.1.bt2l//'` + [ -z "\$INDEX" ] && echo "BT2 index files not found" 1>&2 && exit 1 bowtie2 \\ -x \$INDEX \\ -1 ${reads[0]} \\ diff --git a/tests/modules/bowtie2/align/main.nf b/tests/modules/bowtie2/align/main.nf index 2d3945e5..f91394ff 100644 --- a/tests/modules/bowtie2/align/main.nf +++ b/tests/modules/bowtie2/align/main.nf @@ -32,4 +32,4 @@ workflow test_bowtie2_align_paired_end { BOWTIE2_BUILD ( fasta ) BOWTIE2_ALIGN ( input, BOWTIE2_BUILD.out.index, save_unaligned ) -} +} \ No newline at end of file diff --git a/tests/modules/bowtie2/align/nextflow.config b/tests/modules/bowtie2/align/nextflow.config index 8730f1c4..b4640de7 100644 --- a/tests/modules/bowtie2/align/nextflow.config +++ b/tests/modules/bowtie2/align/nextflow.config @@ -1,5 +1,16 @@ +params { + force_large_index = false +} + process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - +} + +if (params.force_large_index) { + process { + withName: BOWTIE2_BUILD { + ext.args = '--large-index' + } + } } diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 95d48b88..11e4c4e5 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -39,3 +39,46 @@ md5sum: 52be6950579598a990570fbcf5372184 - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 md5sum: e3b4ef343dea4dd571642010a7d09597 + +- name: bowtie2 align single-end large-index + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index + tags: + tags: + - bowtie2 + - bowtie2/align + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/bowtie2/bowtie2/genome.3.bt2 + md5sum: 8952b3e0b1ce9a7a5916f2e147180853 + - path: ./output/bowtie2/bowtie2/genome.2.bt2 + md5sum: 22c284084784a0720989595e0c9461fd + - path: ./output/bowtie2/bowtie2/genome.1.bt2 + md5sum: 07d811cd4e350d56267183d2ac7023a5 + - path: ./output/bowtie2/bowtie2/genome.4.bt2 + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 + md5sum: fda48e35925fb24d1c0785f021981e25 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 + md5sum: 802c26d32b970e1b105032b7ce7348b4 + +- name: bowtie2 align paired-end large-index + command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_paired_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index + tags: + - bowtie2 + - bowtie2/align + files: + - path: ./output/bowtie2/test.bam + - path: ./output/bowtie2/test.bowtie2.log + - path: ./output/bowtie2/bowtie2/genome.3.bt2l + md5sum: 8952b3e0b1ce9a7a5916f2e147180853 + - path: ./output/bowtie2/bowtie2/genome.2.bt2l + md5sum: 22c284084784a0720989595e0c9461fd + - path: ./output/bowtie2/bowtie2/genome.1.bt2l + md5sum: 07d811cd4e350d56267183d2ac7023a5 + - path: ./output/bowtie2/bowtie2/genome.4.bt2l + md5sum: c25be5f8b0378abf7a58c8a880b87626 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l + md5sum: fda48e35925fb24d1c0785f021981e25 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l + md5sum: 802c26d32b970e1b105032b7ce7348b4 From 0f1ee7e50dccd1a8717cd448c518b67fb68c1d09 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 12:04:51 +0100 Subject: [PATCH 10/12] bowtie2|test fixes --- tests/modules/bowtie2/align/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 11e4c4e5..961009b4 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -49,17 +49,17 @@ files: - path: ./output/bowtie2/test.bam - path: ./output/bowtie2/test.bowtie2.log - - path: ./output/bowtie2/bowtie2/genome.3.bt2 + - path: ./output/bowtie2/bowtie2/genome.3.bt2l md5sum: 8952b3e0b1ce9a7a5916f2e147180853 - - path: ./output/bowtie2/bowtie2/genome.2.bt2 + - path: ./output/bowtie2/bowtie2/genome.2.bt2l md5sum: 22c284084784a0720989595e0c9461fd - - path: ./output/bowtie2/bowtie2/genome.1.bt2 + - path: ./output/bowtie2/bowtie2/genome.1.bt2l md5sum: 07d811cd4e350d56267183d2ac7023a5 - - path: ./output/bowtie2/bowtie2/genome.4.bt2 + - path: ./output/bowtie2/bowtie2/genome.4.bt2l md5sum: c25be5f8b0378abf7a58c8a880b87626 - - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2 + - path: ./output/bowtie2/bowtie2/genome.rev.1.bt2l md5sum: fda48e35925fb24d1c0785f021981e25 - - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2 + - path: ./output/bowtie2/bowtie2/genome.rev.2.bt2l md5sum: 802c26d32b970e1b105032b7ce7348b4 - name: bowtie2 align paired-end large-index From 40e6bd9dc16a7a4b63d399cf79f0bb1b7e3927c0 Mon Sep 17 00:00:00 2001 From: Chris Cheshire Date: Tue, 3 May 2022 12:07:04 +0100 Subject: [PATCH 11/12] bowtie2|fix tags error --- tests/modules/bowtie2/align/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/modules/bowtie2/align/test.yml b/tests/modules/bowtie2/align/test.yml index 961009b4..ea9aa72a 100644 --- a/tests/modules/bowtie2/align/test.yml +++ b/tests/modules/bowtie2/align/test.yml @@ -42,7 +42,6 @@ - name: bowtie2 align single-end large-index command: nextflow run ./tests/modules/bowtie2/align -entry test_bowtie2_align_single_end -c ./tests/config/nextflow.config -c ./tests/modules/bowtie2/align/nextflow.config --force_large_index - tags: tags: - bowtie2 - bowtie2/align From 96d4a31cc15e3b20ee6a7df0ad39ca59feeb3a02 Mon Sep 17 00:00:00 2001 From: Nicolas Vannieuwkerke Date: Tue, 3 May 2022 15:31:05 +0200 Subject: [PATCH 12/12] Moved the BED file to the main tuple --- modules/vardictjava/main.nf | 3 +-- tests/modules/vardictjava/main.nf | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/vardictjava/main.nf b/modules/vardictjava/main.nf index 454b86a4..c99301bb 100644 --- a/modules/vardictjava/main.nf +++ b/modules/vardictjava/main.nf @@ -10,8 +10,7 @@ process VARDICTJAVA { 'quay.io/biocontainers/vardict-java:1.8.3--hdfd78af_0' }" input: - tuple val(meta), path(bam), path(bai) - path(bed) + tuple val(meta), path(bam), path(bai), path(bed) tuple path(fasta), path(fasta_fai) output: diff --git a/tests/modules/vardictjava/main.nf b/tests/modules/vardictjava/main.nf index d392358a..e29490f3 100644 --- a/tests/modules/vardictjava/main.nf +++ b/tests/modules/vardictjava/main.nf @@ -9,15 +9,14 @@ workflow test_vardictjava { bam_input_ch = Channel.value([ [ id:'test' ], // meta map file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam'], checkIfExists: true), - file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true) + file(params.test_data['homo_sapiens']['illumina']['test_paired_end_sorted_bam_bai'], checkIfExists: true), + file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true) ]) - bed = Channel.value(file(params.test_data['homo_sapiens']['genome']['genome_bed'], checkIfExists: true)) - reference = Channel.value([ file(params.test_data['homo_sapiens']['genome']['genome_fasta'], checkIfExists: true), file(params.test_data['homo_sapiens']['genome']['genome_fasta_fai'], checkIfExists: true) ]) - VARDICTJAVA ( bam_input_ch, bed, reference ) + VARDICTJAVA ( bam_input_ch, reference ) }