From 9472f2fa2dc2d5575d172db5a42fe19fd8243a8b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 11:11:27 +0100 Subject: [PATCH 1/9] chores: Remove deprecated version of the module --- deprecated/gatk/dict/main.nf | 19 ----------------- deprecated/gatk/dict/meta.yml | 25 ----------------------- deprecated/gatk/dict/test/main.nf | 13 ------------ deprecated/gatk/dict/test/nextflow.config | 2 -- 4 files changed, 59 deletions(-) delete mode 100644 deprecated/gatk/dict/main.nf delete mode 100644 deprecated/gatk/dict/meta.yml delete mode 100644 deprecated/gatk/dict/test/main.nf delete mode 100644 deprecated/gatk/dict/test/nextflow.config diff --git a/deprecated/gatk/dict/main.nf b/deprecated/gatk/dict/main.nf deleted file mode 100644 index c6b74c02..00000000 --- a/deprecated/gatk/dict/main.nf +++ /dev/null @@ -1,19 +0,0 @@ -process gatk_dict { - tag {fasta} - - container 'quay.io/biocontainers/gatk4-spark:4.1.4.1--1' - - input: - path fasta - - output: - path "${fasta.baseName}.dict" - - script: - """ - gatk --java-options "-Xmx${task.memory.giga}g" \ - CreateSequenceDictionary \ - --REFERENCE ${fasta} \ - --OUTPUT ${fasta.baseName}.dict - """ -} diff --git a/deprecated/gatk/dict/meta.yml b/deprecated/gatk/dict/meta.yml deleted file mode 100644 index e03282bb..00000000 --- a/deprecated/gatk/dict/meta.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: gatk dict -description: create a dictionary file from a fasta file -keywords: - - dictionary -tools: - - gatk: - description: | - The GATK toolkit offers a wide variety of tools with a primary focus on variant discovery and genotyping, developed in the Data Sciences Platform at the Broad Institute. - homepage: https://gatk.broadinstitute.org/hc/en-us - documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s - doi: 10.1158/1538-7445.AM2017-3590 -input: - - - - input: - type: file - description: Input fasta file - pattern: "*.{fasta,fa}" -output: - - - - dict: - type: file - description: gatk dictionary file - pattern: "*.{fasta,fa}.{dict}" -authors: - - "@maxulysse" diff --git a/deprecated/gatk/dict/test/main.nf b/deprecated/gatk/dict/test/main.nf deleted file mode 100644 index c1005d37..00000000 --- a/deprecated/gatk/dict/test/main.nf +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env nextflow -nextflow.preview.dsl = 2 -include '../../../tests/functions/check_process_outputs.nf' params(params) -include '../main.nf' params(params) - -// Define input channels -input = '../../../test-datasets/tools/bwa/index/input/reference.fasta' - -// Run the workflow -workflow { - gatk_dict(input) - // .check_output() -} diff --git a/deprecated/gatk/dict/test/nextflow.config b/deprecated/gatk/dict/test/nextflow.config deleted file mode 100644 index c137a138..00000000 --- a/deprecated/gatk/dict/test/nextflow.config +++ /dev/null @@ -1,2 +0,0 @@ -docker.enabled = true -params.outdir = './results' From 8c4d201dd53ac1f5cae3cd0fb064e6eacc7a5d01 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 11:27:25 +0100 Subject: [PATCH 2/9] chores: remove deprecated bwamem2/index modules cf #166 --- deprecated/bwa-mem2/index/main.nf | 16 ------------ deprecated/bwa-mem2/index/meta.yml | 26 ------------------- deprecated/bwa-mem2/index/test/main.nf | 16 ------------ .../bwa-mem2/index/test/nextflow.config | 2 -- 4 files changed, 60 deletions(-) delete mode 100644 deprecated/bwa-mem2/index/main.nf delete mode 100644 deprecated/bwa-mem2/index/meta.yml delete mode 100644 deprecated/bwa-mem2/index/test/main.nf delete mode 100644 deprecated/bwa-mem2/index/test/nextflow.config diff --git a/deprecated/bwa-mem2/index/main.nf b/deprecated/bwa-mem2/index/main.nf deleted file mode 100644 index 7dd459e4..00000000 --- a/deprecated/bwa-mem2/index/main.nf +++ /dev/null @@ -1,16 +0,0 @@ -process bwa-mem2_index { - tag {fasta} - - container 'quay.io/biocontainers/bwa-mem2:2.0--he513fc3_0' - - input: - path(fasta) - - output: - path("${fasta}.*") - - script: - """ - bwa-mem2 index ${fasta} - """ -} diff --git a/deprecated/bwa-mem2/index/meta.yml b/deprecated/bwa-mem2/index/meta.yml deleted file mode 100644 index 97d6e854..00000000 --- a/deprecated/bwa-mem2/index/meta.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: bwa-mem2 index -description: create indexes for BWA from a fasta file -keywords: - - index -tools: - - bwa: - description: | - Bwa-mem2 is the next version of the bwa-mem algorithm in bwa. - It produces alignment identical to bwa and is ~1.3-3.1x faster depending on the use-case, dataset and the running machine. - homepage: https://github.com/bwa-mem2/bwa-mem2 - documentation: https://github.com/bwa-mem2/bwa-mem2 - arxiv: arXiv:1303.3997 -input: - - - - input: - type: file - description: Input fasta file - pattern: "*.{fasta,fa}" -output: - - - - index: - type: file - description: bwa indexes file - pattern: "*.{fasta,fa}.{amb,ann,bwt,pac,sa}" -authors: - - "@maxulysse" diff --git a/deprecated/bwa-mem2/index/test/main.nf b/deprecated/bwa-mem2/index/test/main.nf deleted file mode 100644 index 6acb4d61..00000000 --- a/deprecated/bwa-mem2/index/test/main.nf +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env nextflow -nextflow.preview.dsl = 2 -include '../../../nf-core/module_testing/check_process_outputs.nf' params(params) -include '../main.nf' params(params) - -// Define input channels -input = '../../../test-datasets/tools/bwa/index/input/reference.fasta' -Channel - .from(input) - .set { ch_input } - -// Run the workflow -workflow { - fastqc(ch_input) - // .check_output() -} diff --git a/deprecated/bwa-mem2/index/test/nextflow.config b/deprecated/bwa-mem2/index/test/nextflow.config deleted file mode 100644 index c137a138..00000000 --- a/deprecated/bwa-mem2/index/test/nextflow.config +++ /dev/null @@ -1,2 +0,0 @@ -docker.enabled = true -params.outdir = './results' From 7f35a1742d844c8d91c933a30d70b2e90a73f7c0 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 13:40:06 +0100 Subject: [PATCH 3/9] chores: add gatk/createsequencedictionary module --- .github/filters.yml | 4 ++ .../createsequencedictionary/functions.nf | 59 +++++++++++++++++++ .../gatk/createsequencedictionary/main.nf | 38 ++++++++++++ .../gatk/createsequencedictionary/meta.yml | 51 ++++++++++++++++ .../gatk/createsequencedictionary/main.nf | 9 +++ .../gatk/createsequencedictionary/test.yml | 8 +++ 6 files changed, 169 insertions(+) create mode 100644 software/gatk/createsequencedictionary/functions.nf create mode 100644 software/gatk/createsequencedictionary/main.nf create mode 100644 software/gatk/createsequencedictionary/meta.yml create mode 100644 tests/software/gatk/createsequencedictionary/main.nf create mode 100644 tests/software/gatk/createsequencedictionary/test.yml diff --git a/.github/filters.yml b/.github/filters.yml index 9fe5cdb9..b2f0b04c 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -128,6 +128,10 @@ fastqc: - software/fastqc/** - tests/software/fastqc/** +gatk_createsequencedictionary: + - software/gatk/createsequencedictionary/** + - tests/software/gatk/createsequencedictionary/** + gffread: - software/gffread/** - tests/software/gffread/** diff --git a/software/gatk/createsequencedictionary/functions.nf b/software/gatk/createsequencedictionary/functions.nf new file mode 100644 index 00000000..d25eea86 --- /dev/null +++ b/software/gatk/createsequencedictionary/functions.nf @@ -0,0 +1,59 @@ +/* + * ----------------------------------------------------- + * Utility functions used in nf-core DSL2 module files + * ----------------------------------------------------- + */ + +/* + * Extract name of software tool from process name using $task.process + */ +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + +/* + * Function to initialise default values and to generate a Groovy Map of available options for nf-core modules + */ +def initOptions(Map args) { + def Map options = [:] + options.args = args.args ?: '' + options.args2 = args.args2 ?: '' + options.publish_by_id = args.publish_by_id ?: false + options.publish_dir = args.publish_dir ?: '' + options.publish_files = args.publish_files + options.suffix = args.suffix ?: '' + return options +} + +/* + * Tidy up and join elements of a list to return a path string + */ +def getPathFromList(path_list) { + def paths = path_list.findAll { item -> !item?.trim().isEmpty() } // Remove empty entries + paths = paths.collect { it.trim().replaceAll("^[/]+|[/]+\$", "") } // Trim whitespace and trailing slashes + return paths.join('/') +} + +/* + * Function to save/publish module results + */ +def saveFiles(Map args) { + if (!args.filename.endsWith('.version.txt')) { + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + if (ioptions.publish_by_id) { + path_list.add(args.publish_id) + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" + } + } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" + } + } +} diff --git a/software/gatk/createsequencedictionary/main.nf b/software/gatk/createsequencedictionary/main.nf new file mode 100644 index 00000000..34f04123 --- /dev/null +++ b/software/gatk/createsequencedictionary/main.nf @@ -0,0 +1,38 @@ +// Import generic module functions +include { initOptions; saveFiles; getSoftwareName } from './functions' + +params.options = [:] +def options = initOptions(params.options) + +process GATK_CREATESEQUENCEDICTIONARY { + tag "$fasta" + label 'process_medium' + publishDir "${params.outdir}", + mode: params.publish_dir_mode, + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), publish_id:'') } + + conda (params.enable_conda ? "bioconda::gatk4=4.1.9.0" : null) + if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/gatk4:4.1.9.0--py39_0" + } else { + container "quay.io/biocontainers/gatk4:4.1.9.0--py39_0" + } + + input: + path fasta + + output: + path "*.dict" , emit: dict + path "*.version.txt" , emit: version + + script: + def software = getSoftwareName(task.process) + """ + gatk --java-options "-Xmx${task.memory.toGiga()}g" \\ + CreateSequenceDictionary \\ + --REFERENCE $fasta \\ + $options.args + + echo \$(gatk CreateSequenceDictionary --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt + """ +} diff --git a/software/gatk/createsequencedictionary/meta.yml b/software/gatk/createsequencedictionary/meta.yml new file mode 100644 index 00000000..b53c90eb --- /dev/null +++ b/software/gatk/createsequencedictionary/meta.yml @@ -0,0 +1,51 @@ +name: gatk_createsequencedictionary +description: Creates a sequence dictionary for a reference sequence +keywords: + - dictionary + - fasta +tools: + - gatk: + description: | + Developed in the Data Sciences Platform at the Broad Institute, the toolkit offers a wide variety of tools + with a primary focus on variant discovery and genotyping. Its powerful processing engine + and high-performance computing features make it capable of taking on projects of any size. + homepage: https://gatk.broadinstitute.org/hc/en-us + documentation: https://gatk.broadinstitute.org/hc/en-us/categories/360002369672s + doi: 10.1158/1538-7445.AM2017-3590 +params: + - outdir: + type: string + description: | + The pipeline's output directory. By default, the module will + output files into `$params.outdir/` + - publish_dir_mode: + type: string + description: | + Value for the Nextflow `publishDir` mode parameter. + Available: symlink, rellink, link, copy, copyNoFollow, move. + - enable_conda: + type: boolean + description: | + Run the module with Conda using the software specified + via the `conda` directive + - singularity_pull_docker_container: + type: boolean + description: | + Instead of directly downloading Singularity images for use with Singularity, + force the workflow to pull and convert Docker containers instead. +input: + - fasta: + type: file + description: Input fasta file + pattern: "*.{fasta,fa}" +output: + - bam: + type: file + description: gatk dictionary file + pattern: "*.{dict}" + - version: + type: file + description: File containing software version + pattern: "*.{version.txt}" +authors: + - "@maxulysse" diff --git a/tests/software/gatk/createsequencedictionary/main.nf b/tests/software/gatk/createsequencedictionary/main.nf new file mode 100644 index 00000000..cc366b0b --- /dev/null +++ b/tests/software/gatk/createsequencedictionary/main.nf @@ -0,0 +1,9 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { GATK_CREATESEQUENCEDICTIONARY } from '../../../../software/gatk/createsequencedictionary/main.nf' addParams( options: [:] ) + +workflow test_gatk_createsequencedictionary { + GATK_CREATESEQUENCEDICTIONARY ( file("${launchDir}/tests/data/fasta/E_coli/NC_010473.fa", checkIfExists: true) ) +} \ No newline at end of file diff --git a/tests/software/gatk/createsequencedictionary/test.yml b/tests/software/gatk/createsequencedictionary/test.yml new file mode 100644 index 00000000..4e84a09c --- /dev/null +++ b/tests/software/gatk/createsequencedictionary/test.yml @@ -0,0 +1,8 @@ +- name: gatk createsequencedictionary + command: nextflow run ./tests/software/gatk/createsequencedictionary -entry test_gatk_createsequencedictionary -c tests/config/nextflow.config + tags: + - gatk + - gatk_createsequencedictionary + files: + - path: output/gatk/NC_010473.dict + md5sum: 2b5e6b5d82553ddcaf85156ba856a55c From 053dd6ee1b8dbec9c160e72b1e1a5505a6f35ae6 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 13:40:22 +0100 Subject: [PATCH 4/9] add minimal cpu,memory and time requirements --- tests/config/nextflow.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index baa83417..26e10357 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -5,6 +5,12 @@ params { singularity_pull_docker_container = false } +process { + cpus = 2 + memory = 6.GB + time = 48.h +} + if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true From 30e3f2c5f56ea09cc653c1c06cde3b4192cc5d71 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 13:42:34 +0100 Subject: [PATCH 5/9] fix: ECLint check: trailing whitespace --- software/gatk/createsequencedictionary/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gatk/createsequencedictionary/main.nf b/software/gatk/createsequencedictionary/main.nf index 34f04123..bb1d7e96 100644 --- a/software/gatk/createsequencedictionary/main.nf +++ b/software/gatk/createsequencedictionary/main.nf @@ -22,7 +22,7 @@ process GATK_CREATESEQUENCEDICTIONARY { path fasta output: - path "*.dict" , emit: dict + path "*.dict" , emit: dict path "*.version.txt" , emit: version script: From 28dc9227d709f90b15482f627b263ac27150244c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 13:54:40 +0100 Subject: [PATCH 6/9] fix: remove memory requirements and simplify URI --- software/gatk/createsequencedictionary/main.nf | 9 ++++++++- tests/config/nextflow.config | 6 ------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/software/gatk/createsequencedictionary/main.nf b/software/gatk/createsequencedictionary/main.nf index bb1d7e96..f6350226 100644 --- a/software/gatk/createsequencedictionary/main.nf +++ b/software/gatk/createsequencedictionary/main.nf @@ -27,10 +27,17 @@ process GATK_CREATESEQUENCEDICTIONARY { script: def software = getSoftwareName(task.process) + def avail_mem = 6 + if (!task.memory) { + log.info '[GATK] Available memory not known - defaulting to 6GB. Specify process memory requirements to change this.' + } else { + avail_mem = task.memory.giga + } """ - gatk --java-options "-Xmx${task.memory.toGiga()}g" \\ + gatk --java-options "-Xmx${avail_mem}g" \\ CreateSequenceDictionary \\ --REFERENCE $fasta \\ + --URI $fasta $options.args echo \$(gatk CreateSequenceDictionary --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt diff --git a/tests/config/nextflow.config b/tests/config/nextflow.config index 26e10357..baa83417 100644 --- a/tests/config/nextflow.config +++ b/tests/config/nextflow.config @@ -5,12 +5,6 @@ params { singularity_pull_docker_container = false } -process { - cpus = 2 - memory = 6.GB - time = 48.h -} - if ("$PROFILE" == "singularity") { singularity.enabled = true singularity.autoMounts = true From 763800a0e294c865458d0ac63fca1f1ebd7ae950 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 16 Feb 2021 13:54:54 +0100 Subject: [PATCH 7/9] fix: correct md5sum --- tests/software/gatk/createsequencedictionary/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/software/gatk/createsequencedictionary/test.yml b/tests/software/gatk/createsequencedictionary/test.yml index 4e84a09c..39e4edb0 100644 --- a/tests/software/gatk/createsequencedictionary/test.yml +++ b/tests/software/gatk/createsequencedictionary/test.yml @@ -5,4 +5,4 @@ - gatk_createsequencedictionary files: - path: output/gatk/NC_010473.dict - md5sum: 2b5e6b5d82553ddcaf85156ba856a55c + md5sum: 30b5f2501f6eb68b0270cc5626f5be4c From 7c64e70c9efb42f8bdc449015e7593b40149a119 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 16 Feb 2021 22:07:51 +0000 Subject: [PATCH 8/9] Update software/gatk/createsequencedictionary/meta.yml --- software/gatk/createsequencedictionary/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gatk/createsequencedictionary/meta.yml b/software/gatk/createsequencedictionary/meta.yml index b53c90eb..b192df80 100644 --- a/software/gatk/createsequencedictionary/meta.yml +++ b/software/gatk/createsequencedictionary/meta.yml @@ -39,7 +39,7 @@ input: description: Input fasta file pattern: "*.{fasta,fa}" output: - - bam: + - dict: type: file description: gatk dictionary file pattern: "*.{dict}" From a6d6c0b973883952229b2f2b7f788ae6bff4e8fc Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 16 Feb 2021 22:08:11 +0000 Subject: [PATCH 9/9] Update software/gatk/createsequencedictionary/main.nf --- software/gatk/createsequencedictionary/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gatk/createsequencedictionary/main.nf b/software/gatk/createsequencedictionary/main.nf index f6350226..dd4f418c 100644 --- a/software/gatk/createsequencedictionary/main.nf +++ b/software/gatk/createsequencedictionary/main.nf @@ -37,7 +37,7 @@ process GATK_CREATESEQUENCEDICTIONARY { gatk --java-options "-Xmx${avail_mem}g" \\ CreateSequenceDictionary \\ --REFERENCE $fasta \\ - --URI $fasta + --URI $fasta \\ $options.args echo \$(gatk CreateSequenceDictionary --version 2>&1) | sed 's/^.*(GATK) v//; s/ HTSJDK.*\$//' > ${software}.version.txt