From 3f534c154ea30152983865d10e6d8d13f68c8cf8 Mon Sep 17 00:00:00 2001 From: arontommi Date: Wed, 6 Nov 2019 13:10:09 +0100 Subject: [PATCH 001/181] documentation on how to fix bianca cluster configuration problem --- docs/uppmax.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/uppmax.md b/docs/uppmax.md index 07a7f14..d5bba37 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -26,6 +26,12 @@ If running on Bianca, you will have no internet connection and these configs wil Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. +Please note that Bianca only allocates 7 GB memory per core so max memory needs to be configured: + +```bash +--max_memory "112GB" +``` + ## Getting more memory If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: From a882b6cbaa8bc7542111be1a25a304bbfdc93c39 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Wed, 4 Mar 2020 15:00:33 +0000 Subject: [PATCH 002/181] adding icr_davros config files generic and sarek --- conf/icr_davros.config | 31 +++++++++++++++++++++++++++ conf/pipeline/sarek/icr_davros.config | 8 +++++++ 2 files changed, 39 insertions(+) create mode 100644 conf/icr_davros.config create mode 100644 conf/pipeline/sarek/icr_davros.config diff --git a/conf/icr_davros.config b/conf/icr_davros.config new file mode 100644 index 0000000..73e72ed --- /dev/null +++ b/conf/icr_davros.config @@ -0,0 +1,31 @@ +/* + * ------------------------------------------------- + * Nextflow nf-core config file for ICR davros HPC + * ------------------------------------------------- + * Defines LSF process executor and singularity + * settings. + * + */ + +singularity { + enabled = true + runOptions = "--bind /mnt:/mnt --bind /data:/data" +} +executor { + # This is set because of an issue with too many + # singularity containers launching at once, they + # cause an singularity error with exit code 255. + submitRateLimit = "2 sec" +} +process { + executor = "LSF" +} +params { + # LSF cluster set up with memory tied to cores, + # it can't be requested. Locked at 12G per core. + cpus = 10 + max_cpus = 20 + max_memory = 12.GB + max_time = 168.h + igenomes_base = "/mnt/scratch/DDS/SDDS/BIOINFCO/SHARED/genomes/igenomes" +} \ No newline at end of file diff --git a/conf/pipeline/sarek/icr_davros.config b/conf/pipeline/sarek/icr_davros.config new file mode 100644 index 0000000..9c1ea6a --- /dev/null +++ b/conf/pipeline/sarek/icr_davros.config @@ -0,0 +1,8 @@ +process { + errorStrategy = {task.exitStatus in [104,134,137,139,141,143,255] ? 'retry' : 'finish'} + maxRetries = 5 + withName:MapReads { + memory = {check_resource(12.GB)} + time = {check_resource(48.h * task.attempt)} + } +} \ No newline at end of file From 02d8b81af4a1ca443b5235574fa1cd22756760c1 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Wed, 4 Mar 2020 16:02:11 +0000 Subject: [PATCH 003/181] adding icr_davros as profiles --- nfcore_custom.config | 1 + pipeline/sarek.config | 1 + 2 files changed, 2 insertions(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 72a4ee8..bc2997c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,6 +23,7 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config"} + icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 112aef7..4ef48da 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -11,4 +11,5 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" } + icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" } } \ No newline at end of file From ead67d1e907ce5ab82dd3197005ecb5604e0f6b8 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Thu, 5 Mar 2020 09:24:10 +0000 Subject: [PATCH 004/181] description and contact --- conf/icr_davros.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index 73e72ed..4e5c405 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -6,20 +6,27 @@ * settings. * */ +params { + config_profile_description = "Nextflow nf-core profile for ICR davros HPC" + config_profile_contact = "Adrian Larkeryd" +} singularity { enabled = true runOptions = "--bind /mnt:/mnt --bind /data:/data" } + executor { # This is set because of an issue with too many # singularity containers launching at once, they # cause an singularity error with exit code 255. submitRateLimit = "2 sec" } + process { executor = "LSF" } + params { # LSF cluster set up with memory tied to cores, # it can't be requested. Locked at 12G per core. From 46223a3dbba8ae51e89a4dfbb48d2175eebcb767 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Thu, 5 Mar 2020 10:39:19 +0000 Subject: [PATCH 005/181] remove comments --- conf/icr_davros.config | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index 4e5c405..c6c3024 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -1,11 +1,4 @@ -/* - * ------------------------------------------------- - * Nextflow nf-core config file for ICR davros HPC - * ------------------------------------------------- - * Defines LSF process executor and singularity - * settings. - * - */ + params { config_profile_description = "Nextflow nf-core profile for ICR davros HPC" config_profile_contact = "Adrian Larkeryd" @@ -17,9 +10,7 @@ singularity { } executor { - # This is set because of an issue with too many - # singularity containers launching at once, they - # cause an singularity error with exit code 255. + submitRateLimit = "2 sec" } @@ -28,8 +19,7 @@ process { } params { - # LSF cluster set up with memory tied to cores, - # it can't be requested. Locked at 12G per core. + cpus = 10 max_cpus = 20 max_memory = 12.GB From a67e77bb57095f465b8b0461df2d506c59bc8083 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Thu, 5 Mar 2020 10:42:18 +0000 Subject: [PATCH 006/181] put comments back in correct format --- conf/icr_davros.config | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index c6c3024..e4928ca 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -1,4 +1,11 @@ - +/* + * ------------------------------------------------- + * Nextflow nf-core config file for ICR davros HPC + * ------------------------------------------------- + * Defines LSF process executor and singularity + * settings. + * + */ params { config_profile_description = "Nextflow nf-core profile for ICR davros HPC" config_profile_contact = "Adrian Larkeryd" @@ -10,7 +17,9 @@ singularity { } executor { - + // This is set because of an issue with too many + // singularity containers launching at once, they + // cause an singularity error with exit code 255. submitRateLimit = "2 sec" } @@ -19,7 +28,8 @@ process { } params { - + // LSF cluster set up with memory tied to cores, + // it can't be requested. Locked at 12G per core. cpus = 10 max_cpus = 20 max_memory = 12.GB From e395007ba8a916ec0abc96b5bcfd500ed65431b1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 10 Mar 2020 09:34:30 +0100 Subject: [PATCH 007/181] Add reference to EBC in README docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 755b1d2..3d796ee 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ Currently documentation is available for the following systems: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [EBC](docs/ebc.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) From 8b570507cd65b4e6ac7ccc3cf23adc9f26d080e1 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 30 Mar 2020 16:40:11 +0200 Subject: [PATCH 008/181] update s --- conf/pipeline/sarek/uppmax.config | 2 +- conf/uppmax.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index 1e467dd..3feb2df 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -9,7 +9,7 @@ params { igenomeIgnore = true genomes_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : '/sw/data/uppnex/ToolBox/hg38bundle' } - if (hostname ==~ "r.*") { +if (hostname ==~ "r.*") { params.singleCPUmem = 6400.MB } if (hostname ==~ "i.*") { diff --git a/conf/uppmax.config b/conf/uppmax.config index 9eb45fc..037070a 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -26,7 +26,7 @@ params { def hostname = "hostname".execute().text.trim() -if (hostname ==~ "b.*") { +if (hostname ==~ "b.*" || hostname ==~ "s.*") { params.max_memory = 109.GB } From c8de9be7014ee85359caddc04aefd725974d5d6b Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:17:24 +0200 Subject: [PATCH 009/181] Adding UPPMAX specific config for ampliseq --- conf/pipeline/ampliseq/uppmax.config | 15 +++++++++++++++ docs/pipeline/ampliseq/uppmax.md | 17 +++++++++++++++++ pipeline/ampliseq.config | 6 +++++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 conf/pipeline/ampliseq/uppmax.config create mode 100644 docs/pipeline/ampliseq/uppmax.md diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config new file mode 100644 index 0000000..36a1c3b --- /dev/null +++ b/conf/pipeline/ampliseq/uppmax.config @@ -0,0 +1,15 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'Daniel Lundin (daniel.lundin@lnu.se)' + config_profile_description = 'nf-core/ampliseq UPPMAX profile provided by nf-core/configs' +} + +withName: make_SILVA_132_16S_classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } +} + +withName: classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } +} diff --git a/docs/pipeline/ampliseq/uppmax.md b/docs/pipeline/ampliseq/uppmax.md new file mode 100644 index 0000000..7061bdc --- /dev/null +++ b/docs/pipeline/ampliseq/uppmax.md @@ -0,0 +1,17 @@ +# nf-core/configs: uppmax ampliseq specific configuration + +Extra specific configuration for the ampliseq pipeline. + +## Usage + +To use, run the pipeline with `-profile uppmax`. + +This will download and launch the ampliseq specific [`uppmax.config`](../../../conf/pipeline/ampliseq/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX cluster. + +Example: `nextflow run nf-core/ampliseq -profile uppmax` + +## ampliseq specific configurations for uppmax + +Specific configurations for UPPMAX has been made for ampliseq. + +* Makes sure that a fat node is allocated for training and applying a Bayesian classifier. diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 6009d68..7b36722 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -10,4 +10,8 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } -} \ No newline at end of file +} + +profiles { + uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } +} From 79ef94de4782f17b154dce6821e50c9e36fb8697 Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:04 +0200 Subject: [PATCH 010/181] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 7b36722..167877c 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -12,6 +12,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } } -profiles { uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From 5a557ed7d5b08091eb9f89c6df4c4439c73dbeaa Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:12 +0200 Subject: [PATCH 011/181] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index 167877c..c86e152 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -11,6 +11,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } } - uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From e0f9374272c2e97954b611fa3738de70d3e076ba Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:33:19 +0200 Subject: [PATCH 012/181] Update pipeline/ampliseq.config Co-Authored-By: Maxime Garcia --- pipeline/ampliseq.config | 1 - 1 file changed, 1 deletion(-) diff --git a/pipeline/ampliseq.config b/pipeline/ampliseq.config index c86e152..9cb19f7 100644 --- a/pipeline/ampliseq.config +++ b/pipeline/ampliseq.config @@ -10,6 +10,5 @@ profiles { binac { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/binac.config" } -} uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/ampliseq/uppmax.config" } } From e77cd380929e33a430ef62925cdbb81dd4f8ed50 Mon Sep 17 00:00:00 2001 From: Daniel Lundin Date: Tue, 31 Mar 2020 15:42:25 +0200 Subject: [PATCH 013/181] Adding ampliseq/UPPMAX doc to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95c1d79..49283ed 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,7 @@ Currently documentation is available for the following pipeline within the speci * ampliseq * [BINAC](docs/pipeline/ampliseq/binac.md) + * [UPPMAX](docs/pipeline/ampliseq/uppmax.md) * eager * [SHH](docs/pipeline/eager/shh.md) * rnafusion From 82965d25c8e431f018a6d6373c7cc48dd08d4ae6 Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Wed, 8 Apr 2020 11:54:34 +0100 Subject: [PATCH 014/181] hostname warning for davros --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index bc2997c..a9c0eac 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -48,6 +48,7 @@ params { hostnames = [ cbe: ['.cbe.vbc.ac.at'], crick: ['.thecrick.org'], + icr_davros: ['.davros.compute.estate'], genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] From 78be4b358ee2609f56e8857391df3c3c157dbb1a Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Wed, 8 Apr 2020 11:55:16 +0100 Subject: [PATCH 015/181] icr_davros igenomes base --- conf/icr_davros.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index e4928ca..7ba51ff 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -34,5 +34,5 @@ params { max_cpus = 20 max_memory = 12.GB max_time = 168.h - igenomes_base = "/mnt/scratch/DDS/SDDS/BIOINFCO/SHARED/genomes/igenomes" + igenomes_base = "/mnt/scratch/readonly/igenomes" } \ No newline at end of file From 405801638a89fdce078c8a8237ba660d0b6bea5b Mon Sep 17 00:00:00 2001 From: Adrian Larkeryd Date: Wed, 8 Apr 2020 11:56:17 +0100 Subject: [PATCH 016/181] icr_davros sarek config description --- conf/pipeline/sarek/icr_davros.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/pipeline/sarek/icr_davros.config b/conf/pipeline/sarek/icr_davros.config index 9c1ea6a..8051337 100644 --- a/conf/pipeline/sarek/icr_davros.config +++ b/conf/pipeline/sarek/icr_davros.config @@ -1,3 +1,8 @@ +/* + * ------------------------------------------------- + * Nextflow nf-core config file for ICR davros HPC + * ------------------------------------------------- + */ process { errorStrategy = {task.exitStatus in [104,134,137,139,141,143,255] ? 'retry' : 'finish'} maxRetries = 5 From 34dc18faf6d876a82372e00af7159c954a1dcf34 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 9 Apr 2020 11:35:37 +0200 Subject: [PATCH 017/181] Update uppmax.config --- conf/pipeline/sarek/uppmax.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index 3feb2df..df2f3a2 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -9,9 +9,13 @@ params { igenomeIgnore = true genomes_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : '/sw/data/uppnex/ToolBox/hg38bundle' } + +def hostname = "hostname".execute().text.trim() + if (hostname ==~ "r.*") { params.singleCPUmem = 6400.MB } + if (hostname ==~ "i.*") { params.singleCPUmem = 15.GB } From 19316ce8527fb1f1f46c842c65116d933779e0bd Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 13:40:35 +0200 Subject: [PATCH 018/181] add viralrecon specific config --- pipeline/viralrecon.config | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 pipeline/viralrecon.config diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config new file mode 100644 index 0000000..3aa23af --- /dev/null +++ b/pipeline/viralrecon.config @@ -0,0 +1,16 @@ +/* + * ------------------------------------------------- + * nfcore/viralrecon custom profile Nextflow config file + * ------------------------------------------------- + * Defines viral reference genomes for all environments. + */ + +params { + // Genome reference file paths + genomes { + 'NC_045512.2' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.fasta.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" + } + } +} From 290d0a21803dc2baee983aa358e7260b97de642c Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 13:31:20 +0100 Subject: [PATCH 019/181] Update pipeline/viralrecon.config --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 3aa23af..afe373c 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -9,7 +9,7 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.fasta.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" } } From 4fd9b73cecef10a2e863d88b5bd753ec5a4fd4a7 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 13:31:39 +0100 Subject: [PATCH 020/181] Update pipeline/viralrecon.config --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index afe373c..45c8247 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -10,7 +10,7 @@ params { genomes { 'NC_045512.2' { fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/reference/NC_045512.2.gff.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" } } } From 50a44d037bb86703f405c5903ffa944222ce0052 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 17:02:36 +0200 Subject: [PATCH 021/181] better docs and organisation of giles --- README.md | 12 ++++++++++-- conf/pipeline/viralrecon/genomes.config | 16 ++++++++++++++++ docs/pipeline/viralrecon/genomes.md | 9 +++++++++ pipeline/viralrecon.config | 11 +---------- 4 files changed, 36 insertions(+), 12 deletions(-) create mode 100644 conf/pipeline/viralrecon/genomes.config create mode 100644 docs/pipeline/viralrecon/genomes.md diff --git a/README.md b/README.md index 49283ed..17ce78c 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A repository for hosting Nextflow configuration files containing custom paramete * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) * [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) * [Pipeline-specific documentation](#pipeline-specific-documentation) * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) @@ -159,9 +160,9 @@ Each configuration file will add new params and overwrite the params already exi Note that pipeline-specific configs are not required and should only be added if needed. -### Pipeline-specific documentation +### Pipeline-specific institutional documentation -Currently documentation is available for the following pipeline within the specific profile: +Currently documentation is available for the following pipelines within specific profiles: * ampliseq * [BINAC](docs/pipeline/ampliseq/binac.md) @@ -174,6 +175,13 @@ Currently documentation is available for the following pipeline within the speci * [MUNIN](docs/pipeline/sarek/munin.md) * [UPPMAX](docs/pipeline/sarek/uppmax.md) +### Pipeline-specific documentation + +Currently documentation is available for the following pipeline: + +* viralrecon + * [genomes](docs/pipeline/viralrecon/genomes.md) + ### Enabling pipeline-specific configs within a pipeline :warning: **This has to be done on a fork of the `nf-core/` repository.** diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config new file mode 100644 index 0000000..45c8247 --- /dev/null +++ b/conf/pipeline/viralrecon/genomes.config @@ -0,0 +1,16 @@ +/* + * ------------------------------------------------- + * nfcore/viralrecon custom profile Nextflow config file + * ------------------------------------------------- + * Defines viral reference genomes for all environments. + */ + +params { + // Genome reference file paths + genomes { + 'NC_045512.2' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + } + } +} diff --git a/docs/pipeline/viralrecon/genomes.md b/docs/pipeline/viralrecon/genomes.md new file mode 100644 index 0000000..9577434 --- /dev/null +++ b/docs/pipeline/viralrecon/genomes.md @@ -0,0 +1,9 @@ +# nf-core/configs: viralrecon specific configuration + +Extra specific configuration for viralrecon pipeline + +## Usage + +Will be used automatically when running the pipeline with the shared configs in the nf-core/configs repository + +This will download and launch the viralrecon specific [`viralrecon.config`](../../../conf/pipeline/viralrecon/genomes.config) which has been pre-configured with custom genomes. diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 45c8247..25d93c4 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -2,15 +2,6 @@ * ------------------------------------------------- * nfcore/viralrecon custom profile Nextflow config file * ------------------------------------------------- - * Defines viral reference genomes for all environments. */ -params { - // Genome reference file paths - genomes { - 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" - } - } -} +includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" } \ No newline at end of file From 5678494e45eb098cdc59a05b83c71d6d9d1d63ab Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 9 Apr 2020 17:13:19 +0200 Subject: [PATCH 022/181] fix typo --- pipeline/viralrecon.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 25d93c4..3a79bef 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -4,4 +4,4 @@ * ------------------------------------------------- */ -includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" } \ No newline at end of file +includeConfig "${params.custom_config_base}/conf/pipeline/viralrecon/genomes.config" \ No newline at end of file From 6834f020cb45fdfb1f4bcf181ced275830b4fbb7 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 17:49:15 +0100 Subject: [PATCH 023/181] Update viralrecon.config --- pipeline/viralrecon.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pipeline/viralrecon.config b/pipeline/viralrecon.config index 45c8247..d889460 100644 --- a/pipeline/viralrecon.config +++ b/pipeline/viralrecon.config @@ -9,8 +9,12 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz" + } + 'MN908947.3' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz" } } } From 2300d5a9626a486c41248f751d22e39f1ae7e717 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 9 Apr 2020 18:22:41 +0100 Subject: [PATCH 024/181] Update genomes.config --- conf/pipeline/viralrecon/genomes.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 45c8247..d889460 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -9,8 +9,12 @@ params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.fasta.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/NC_045512.2.gff.gz" + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz" + } + 'MN908947.3' { + fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz" + gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz" } } } From 1e68a10ef024487014d0e989f9362208c54daa03 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 28 Apr 2020 16:02:26 +0100 Subject: [PATCH 025/181] adding documentation for icr_davros --- docs/icr_davros.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/icr_davros.md diff --git a/docs/icr_davros.md b/docs/icr_davros.md new file mode 100644 index 0000000..93bcf06 --- /dev/null +++ b/docs/icr_davros.md @@ -0,0 +1,22 @@ +# nf-core/configs: Institute of Cancer Research (Davros HPC) Configuration + +Deployment and testing of nf-core pipelines at the Davros cluster is on-going. + +To run an nf-core pipeline on Davros, run the pipeline with `-profile icr_davros`. This will download and launch the [`icr_davros.config`](../conf/icr_davros.config) which has been pre-configured with a setup suitable for the Davros HPC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow using the environment module system. You can do this by issuing the commands below: + +```bash +## Load Nextflow environment modules +module load Nextflow/19.10.0 +``` + +Singularity is installed on the compute nodes of Davros, but not the login nodes. There is no module for Singularity. + +A subset of the [AWS-iGenomes](https://github.com/ewels/AWS-iGenomes) resource has been made available locally on Davros so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline you want to execute. You can do this by simply using the `--genome ` parameter. Some of the more exotic genomes may not have been downloaded onto Davros so have a look in the `igenomes_base` path specified in [`icr_davros.config`](../conf/icr_davros.config), and if your genome of interest isn't present please contact [Scientific Computing](mailto:schelpdesk@icr.ac.uk). + +Alternatively, if you are running the pipeline regularly for genomes that arent available in the iGenomes resource, we recommend creating a config file with paths to your reference genome indices (see [`reference genomes documentation`](https://nf-co.re/usage/reference_genomes) for instructions). + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large. All of the main output files will be saved in the `results/` directory. + +>NB: Nextflow will need to submit the jobs via LSF to the HPC cluster. This can be done from an interactive or normal job. If in doubt contact Scientific Computing. From ad12bd4ad13eea61c152209241c2ca928846fcf4 Mon Sep 17 00:00:00 2001 From: Adrian Date: Tue, 28 Apr 2020 16:03:06 +0100 Subject: [PATCH 026/181] testing automounts --- conf/icr_davros.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index 7ba51ff..9aa572a 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -8,12 +8,13 @@ */ params { config_profile_description = "Nextflow nf-core profile for ICR davros HPC" - config_profile_contact = "Adrian Larkeryd" + config_profile_contact = "Adrian Larkeryd (@adrlar)" } singularity { enabled = true - runOptions = "--bind /mnt:/mnt --bind /data:/data" + // runOptions = "--bind /mnt:/mnt --bind /data:/data" + autoMounts = true } executor { @@ -35,4 +36,4 @@ params { max_memory = 12.GB max_time = 168.h igenomes_base = "/mnt/scratch/readonly/igenomes" -} \ No newline at end of file +} From e6f4d58fa60aa0ae3256d0ab98d5b862573485f9 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 28 Apr 2020 17:06:04 +0200 Subject: [PATCH 027/181] update rnafusion munin specific config --- conf/pipeline/rnafusion/munin.config | 52 +--------------------------- docs/pipeline/rnafusion/munin.md | 1 - pipeline/rnafusion.config | 13 +++++++ 3 files changed, 14 insertions(+), 52 deletions(-) create mode 100644 pipeline/rnafusion.config diff --git a/conf/pipeline/rnafusion/munin.config b/conf/pipeline/rnafusion/munin.config index 549d957..b18f6ad 100644 --- a/conf/pipeline/rnafusion/munin.config +++ b/conf/pipeline/rnafusion/munin.config @@ -4,57 +4,7 @@ params { max_cpus = 24 max_memory = 256.GB max_time = 72.h - container_version = '1.1.0' - - // Tool versions - arriba_version = '1.1.0' - ericscript_version = '0.5.5' - fusioncatcher_version = '1.20' - fusion_inspector_version = '1.3.1-star2.7.0f' - pizzly_version = '0.37.3' - squid_version = '1.5-star2.7.0f' - star_fusion_version = '1.6.0' // Paths - reference_base = '/data1/references/rnafusion' - containerPath = "file:///data1/containers/rnafusion_containers_v${container_version}" - containerPathCommon = "file:///data1/containers/rnafusion_common" - - // References - fasta = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.all.fa" - gtf = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.gtf" - transcript = "${params.reference_base}/1.1.0/Homo_sapiens.GRCh38_r97.cdna.all.fa.gz" - databases = "${params.reference_base}/1.1.0/databases" - - star_index = "${params.reference_base}/1.1.0/star_index/star" - arriba_ref = "${params.reference_base}/arriba_ref" - fusioncatcher_ref = "${params.reference_base}/fusioncatcher_ref/human_v98" - star_fusion_ref = "${params.reference_base}/star_fusion/1.6.0/ensembl/ctat_genome_lib_build_dir" - ericscript_ref = "${params.reference_base}/ericscript_ref/ericscript_db_homosapiens_ensembl84" -} - -process { - container = "${params.containerPath}/rnafusion_1.1.0.img" - - withName: "arriba|arriba_visualization" { - container = "${params.containerPath}/rnafusion_arriba_v${params.arriba_version}.img" - } - withName: "star_fusion|download_star_fusion" { - container = "${params.containerPath}/rnafusion_star-fusion_v${params.star_fusion_version}.img" - } - withName:fusioncatcher { - container = "${params.containerPath}/rnafusion_fusioncatcher_v${params.fusioncatcher_version}.img" - } - withName:fusion_inspector { - container = "${params.containerPath}/rnafusion_fusion-inspector_v${params.fusion_inspector_version}.img" - } - withName:ericscript { - container = "${params.containerPathCommon}/rnafusion_ericscript_v${params.ericscript_version}.img" - } - withName:pizzly { - container = "${params.containerPathCommon}/rnafusion_pizzly_v${params.pizzly_version}.img" - } - withName:squid { - container = "${params.containerPath}/rnafusion_squid_v${params.squid_version}.img" - } + genomes_base = '/data1/references/rnafusion/dev/' } diff --git a/docs/pipeline/rnafusion/munin.md b/docs/pipeline/rnafusion/munin.md index daa1dd1..607db9e 100644 --- a/docs/pipeline/rnafusion/munin.md +++ b/docs/pipeline/rnafusion/munin.md @@ -15,5 +15,4 @@ Example: `nextflow run nf-core/rnafusion -profile munin` Specific configurations for `MUNIN` has been made for rnafusion. * `cpus`, `memory` and `time` max requirements. -* Paths to specific containers * Paths to specific references and indexes diff --git a/pipeline/rnafusion.config b/pipeline/rnafusion.config new file mode 100644 index 0000000..2d86d89 --- /dev/null +++ b/pipeline/rnafusion.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/rnafusion custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/rnafusion folder and imported + * under a profile name here. + */ + +profiles { + munin { includeConfig "${params.custom_config_base}/conf/pipeline/rnafusion/munin.config" } +} \ No newline at end of file From d6813d9c39b8b5458cca8df032957843f554d643 Mon Sep 17 00:00:00 2001 From: FriederikeHanssen Date: Mon, 4 May 2020 17:09:43 +0200 Subject: [PATCH 028/181] Add scratch = true to avoid writing of intermediate file to /tmp --- conf/cfc.config | 1 + conf/cfc_dev.config | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/cfc.config b/conf/cfc.config index 62d3626..ffe0395 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -14,6 +14,7 @@ process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + scratch = 'true' } weblog{ diff --git a/conf/cfc_dev.config b/conf/cfc_dev.config index a7ebf2f..6190314 100644 --- a/conf/cfc_dev.config +++ b/conf/cfc_dev.config @@ -13,6 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + scratch = 'true' } weblog{ From d42d7f41463d26f97308a4c6c396e71bf998dd43 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 4 May 2020 21:02:16 +0200 Subject: [PATCH 029/181] Sets the bwa aln parameter to 0.01 which is more commonly used at SHH --- conf/pipeline/eager/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index a34f0e2..22a60d4 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -17,3 +17,7 @@ process { queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } } + +params { + bwaalnn = 0.01 +} From 4804052b23f607ebeb2ff160abed6d669baa7420 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 4 May 2020 21:04:17 +0200 Subject: [PATCH 030/181] Update SHH nf-core/eager pipeline docs --- docs/pipeline/eager/shh.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index a0dc092..6112842 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -15,3 +15,4 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. +* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.03, to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. From 4d7e9ccf2f2548f0538d3693e1be31c2084cd2c1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 6 May 2020 08:35:23 +0200 Subject: [PATCH 031/181] Apply suggestions from review --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 3d796ee..755b1d2 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,6 @@ Currently documentation is available for the following systems: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) -* [EBC](docs/ebc.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) From cf6e697f0b6e4ba6eb7a5ff0671d2a347649b1f1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 09:16:08 +0200 Subject: [PATCH 032/181] Further update aln based on modifications of EAGER defaults --- conf/pipeline/eager/shh.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 22a60d4..b07004d 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -20,4 +20,5 @@ process { params { bwaalnn = 0.01 + bwaalnl = 32 } From 0a312cd2c91dfa397bcf23b67ac4867fca88e7dc Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 09:17:56 +0200 Subject: [PATCH 033/181] Update shh.md --- docs/pipeline/eager/shh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index 6112842..41d0893 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -15,4 +15,4 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. -* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.03, to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. +* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.04, and `-l` to 32 (rather than 1024) to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. From 2e3b577fb3f49dc574edcbdd55fd3e95d2127dd3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 7 May 2020 11:50:11 +0200 Subject: [PATCH 034/181] Typo fixes I'll never forgive our directory for that name... --- docs/shh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/shh.md b/docs/shh.md index c780f21..6f61426 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de). -To use, run the pipeline with `-profile ssh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile ssh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: +To use, run the pipeline with `-profile shh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile shh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: ```bash /projects1/singularity_scratch/cache/ @@ -10,7 +10,7 @@ To use, run the pipeline with `-profile ssh`. You can further with optimise subm however this will likely change to a read-only directory in the future that will be managed by the IT team. -This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile ssh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile shh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team. >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. From 4814bac1c342e9715539a28a0967b33d3df5e86d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:42:42 +0200 Subject: [PATCH 035/181] Fix sub-profiles --- conf/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index a6106be..852f1d3 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -33,11 +33,15 @@ params { profiles { cdag { + params { config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' + } } sdag { + params { config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 + } } } From 8225e07819fbfc76aa1a4048e12d740c52181aff Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:47:25 +0200 Subject: [PATCH 036/181] Condensed base params and added names for debugging --- conf/shh.config | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 852f1d3..f10f1f8 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,8 +1,13 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' + config_profile_description = 'JFY Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' + max_memory = 256.GB + max_cpus = 32 + max_time = 720.h + //Illumina iGenomes reference file path + igenomes_base = "/projects1/public_data/igenomes/" } cleanup = true @@ -23,23 +28,15 @@ executor { queueSize = 16 } -params { - max_memory = 256.GB - max_cpus = 32 - max_time = 720.h - //Illumina iGenomes reference file path - igenomes_base = "/projects1/public_data/igenomes/" -} - profiles { cdag { params { - config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' + config_profile_description = 'JFY MPI-SHH CDAG profile, provided by nf-core/configs.' } } sdag { params { - config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' + config_profile_description = 'JFY MPI-SHH SDAG profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 } From ff915ee5045345075b854712694408a7e555c0ce Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 12 May 2020 12:47:59 +0200 Subject: [PATCH 037/181] Update shh.config --- conf/pipeline/eager/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index b07004d..1ed9112 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -3,7 +3,7 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' + config_profile_description = 'JFY nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" } From e31e781f2938fc95adc91b4c22f79eacbc4e56e8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:07:06 +0200 Subject: [PATCH 038/181] Remove debugging --- conf/shh.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index f10f1f8..8defc1f 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'JFY Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' + config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' max_memory = 256.GB @@ -31,12 +31,12 @@ executor { profiles { cdag { params { - config_profile_description = 'JFY MPI-SHH CDAG profile, provided by nf-core/configs.' + config_profile_description = 'CDAG MPI-SHH profile, provided by nf-core/configs.' } } sdag { params { - config_profile_description = 'JFY MPI-SHH SDAG profile, provided by nf-core/configs.' + config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 } From d070ed70d18bc3766fb71d053d898e022aae04cb Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:11:42 +0200 Subject: [PATCH 039/181] Update BWA settings based on feedback --- conf/pipeline/eager/shh.config | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 1ed9112..94a0403 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -3,8 +3,12 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'JFY nf-core/eager SHH profile provided by nf-core/configs' + config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" + + // default BWA + bwaalnn = 0.04 + bwaalnl = 32 } // Specific nf-core/eager process configuration @@ -18,7 +22,26 @@ process { } } -params { - bwaalnn = 0.01 - bwaalnl = 32 +profiles { + pathogen_loose { + params { + config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16 + } + } + pathogen_strict { + params { + config_profile_description = 'Pathogen (strict) MPI-SHH SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.1 + bwaalnl = 32 + } + } + human { + params { + config_profile_description = 'Human MPI-SHH SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16500 + } + } } From 0943ba088a2fdd23aabccb73ba96b41001f7d2de Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 May 2020 11:14:42 +0200 Subject: [PATCH 040/181] Mention SHH EAGER group-specific profiles --- docs/pipeline/eager/shh.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index 41d0893..d5fbcb3 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -14,5 +14,5 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. -* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. -* Modifies the bwa aln `-n` parameter to 0.01, rather than typical default of 0.04, and `-l` to 32 (rather than 1024) to syncronise the (unpublished) parameter with EAGER1 and typical/expected usage within the department. +* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG. +* Provides additional group specific profiles, which adapt the `bwa aln` mapping parameters to each context: `pathogens_loose` (`-l 0.01 -n 16`), `pathogens_strict` (`-l 32, -n 0.1`) and `human` (`-l 16500, -n 0.01`). From 1354ae40635dbf130f92df22d93a01e70b07c745 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:19:15 +0200 Subject: [PATCH 041/181] add bi config --- .github/workflows/main.yml | 2 +- conf/bi.config | 11 +++++++++++ nfcore_custom.config | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 conf/bi.config diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bdc485b..b927957 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/conf/bi.config b/conf/bi.config new file mode 100644 index 0000000..874d9f8 --- /dev/null +++ b/conf/bi.config @@ -0,0 +1,11 @@ +params.globalConfig = determine_global_config() + +includeConfig params.globalConfig + +def determine_global_config() { + if( System.getenv('NXF_GLOBAL_CONFIG') == null) + { + throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + } + return System.getenv('NXF_GLOBAL_CONFIG') +} diff --git a/nfcore_custom.config b/nfcore_custom.config index ea2238a..3f89d83 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,6 +11,7 @@ //Please use a new line per include Config section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } + bi { includeConfig "${params.custom_config_base}/conf/bi.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } From c31b03108c4308c70a6f4ac68840ad0cbe37e308 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:22:56 +0200 Subject: [PATCH 042/181] retry without Exception --- conf/bi.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index 874d9f8..091092c 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -3,9 +3,9 @@ params.globalConfig = determine_global_config() includeConfig params.globalConfig def determine_global_config() { - if( System.getenv('NXF_GLOBAL_CONFIG') == null) - { - throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - } + // if( System.getenv('NXF_GLOBAL_CONFIG') == null) + // { + // throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + // } return System.getenv('NXF_GLOBAL_CONFIG') } From 2685275952076be368b7103b0f49bf11f2ed1aac Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:24:47 +0200 Subject: [PATCH 043/181] switch include off --- conf/bi.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/bi.config b/conf/bi.config index 091092c..6fa20d6 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,6 +1,6 @@ params.globalConfig = determine_global_config() -includeConfig params.globalConfig +// includeConfig params.globalConfig def determine_global_config() { // if( System.getenv('NXF_GLOBAL_CONFIG') == null) From 66f191a1cfd87ac5b5b728a0ebc73b84b9d469c9 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:30:00 +0200 Subject: [PATCH 044/181] declate NXF_GLOBAL_CONFIG in ci --- .github/workflows/main.yml | 2 ++ conf/bi.config | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b927957..8226dac 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,9 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + touch /usr/local/bin/test.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' + NXF_GLOBAL_CONFIG: /usr/local/bin/test.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} diff --git a/conf/bi.config b/conf/bi.config index 6fa20d6..874d9f8 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,11 +1,11 @@ params.globalConfig = determine_global_config() -// includeConfig params.globalConfig +includeConfig params.globalConfig def determine_global_config() { - // if( System.getenv('NXF_GLOBAL_CONFIG') == null) - // { - // throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - // } + if( System.getenv('NXF_GLOBAL_CONFIG') == null) + { + throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + } return System.getenv('NXF_GLOBAL_CONFIG') } From ab3070582e94ded2270dcacd69b1348aceec488f Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:32:19 +0200 Subject: [PATCH 045/181] Fix paths to dummy.config --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8226dac..8c6d23b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,9 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - touch /usr/local/bin/test.config + touch ~/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: /usr/local/bin/test.config + NXF_GLOBAL_CONFIG: ~/dummy.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From ff5fdd097bf418b1cb78a7a171c1fb9a4744ee59 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:35:55 +0200 Subject: [PATCH 046/181] debug --- .github/workflows/main.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c6d23b..d4e0232 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,13 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + echo ~ touch ~/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' NXF_GLOBAL_CONFIG: ~/dummy.config - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + run: | + echo $SCRATCH + echo $NXF_GLOBAL_CONFIG + nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From ac47d020cb8f09dac8ceae1ca3e2f86c3ed591f7 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:38:11 +0200 Subject: [PATCH 047/181] debug: continue --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4e0232..8c50e7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,12 +23,11 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - echo ~ - touch ~/dummy.config + touch /home/runner/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: ~/dummy.config + NXF_GLOBAL_CONFIG: /home/runner/dummy.config run: | echo $SCRATCH echo $NXF_GLOBAL_CONFIG From 1a9db371c380e4c1a044a3586bb33ac492f714bd Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:43:32 +0200 Subject: [PATCH 048/181] remove echos --- .github/workflows/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8c50e7a..c906cf9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,4 @@ jobs: env: SCRATCH: '~' NXF_GLOBAL_CONFIG: /home/runner/dummy.config - run: | - echo $SCRATCH - echo $NXF_GLOBAL_CONFIG - nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 3ad0003979a4d7c598be03980d1640c9327a8478 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:46:08 +0200 Subject: [PATCH 049/181] test without dummy config --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c906cf9..0742e5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,9 +23,8 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - touch /home/runner/dummy.config - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: /home/runner/dummy.config + NXF_GLOBAL_CONFIG: ${GITHUB_WORKSPACE}/config/awsbatch.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From f5e2a9f9f3443ae4be1885cd816ae0bc62cc27fc Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 16:55:37 +0200 Subject: [PATCH 050/181] try without dummy.config: use relative path --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0742e5c..0609a85 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,5 +26,5 @@ jobs: - name: Check ${{ matrix.profile }} profile env: SCRATCH: '~' - NXF_GLOBAL_CONFIG: ${GITHUB_WORKSPACE}/config/awsbatch.config + NXF_GLOBAL_CONFIG: awsbatch.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 18d8edb0469c81838ddd4454b2d410b3807fe1cc Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 17:41:19 +0200 Subject: [PATCH 051/181] add description and profile --- conf/bi.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/bi.config b/conf/bi.config index 874d9f8..0ad15d4 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -1,3 +1,8 @@ +params{ + config_profile_description = 'Boehringer Ingelheim internal profile provided by nf-core/configs.' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://www.boehringer-ingelheim.com/' +} params.globalConfig = determine_global_config() includeConfig params.globalConfig From 3c695ba71c87955f9303151b3b77cab64325fe40 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 18:00:47 +0200 Subject: [PATCH 052/181] test error display --- conf/bi.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/bi.config b/conf/bi.config index 0ad15d4..b96c8e9 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -10,6 +10,7 @@ includeConfig params.globalConfig def determine_global_config() { if( System.getenv('NXF_GLOBAL_CONFIG') == null) { + System.err.println("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") } return System.getenv('NXF_GLOBAL_CONFIG') From f139750bf442aefcefe6d9588206037c92757ba5 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Wed, 27 May 2020 19:14:43 +0200 Subject: [PATCH 053/181] print the error message before throwing Exception --- conf/bi.config | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index b96c8e9..f9dfb27 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -10,8 +10,9 @@ includeConfig params.globalConfig def determine_global_config() { if( System.getenv('NXF_GLOBAL_CONFIG') == null) { - System.err.println("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") - throw new Exception("Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file") + def errorMessage = "ERROR: Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file." + System.err.println(errorMessage) + throw new Exception(errorMessage) } return System.getenv('NXF_GLOBAL_CONFIG') } From 0efa9f248ee616bf36347cb37edcb817bbd57f4c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:12:50 +0200 Subject: [PATCH 054/181] Add BI to main readme.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 17ce78c..e5b199c 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Currently documentation is available for the following systems: * [AWSBATCH](docs/awsbatch.md) * [BIGPURPLE](docs/bigpurple.md) +* [BI](docs/bi.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) From f4df5b5a75d4f8541fe04e19aff0e12253bde6a9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:16:22 +0200 Subject: [PATCH 055/181] Add BI Docs --- docs/bi.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/bi.md diff --git a/docs/bi.md b/docs/bi.md new file mode 100644 index 0000000..b256805 --- /dev/null +++ b/docs/bi.md @@ -0,0 +1,9 @@ +# nf-core/configs: BI Configuration + +All nf-core pipelines have been successfully configured for use at Boehringer Ingelheim. + +To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From dd56355c7ccb2732e9afaaafd07a818f8f1143f3 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 28 May 2020 10:18:16 +0200 Subject: [PATCH 056/181] Update bi.md No trailing spaces it said. --- docs/bi.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/bi.md b/docs/bi.md index b256805..450d0df 100644 --- a/docs/bi.md +++ b/docs/bi.md @@ -2,8 +2,8 @@ All nf-core pipelines have been successfully configured for use at Boehringer Ingelheim. -To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile bi`. This will download and launch the [`bi.config`](../conf/bi.config) which has been pre-configured with a setup suitable for the BI systems. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. +Before running the pipeline you will need to follow the internal documentation to run Nextflow on our systems. Similar to that, you need to set an environment variable `NXF_GLOBAL_CONFIG` to the path of the internal global config which is not publicly available here. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 3e4db8c845c5e6434edecd62bcc739291f8e7864 Mon Sep 17 00:00:00 2001 From: Adrian Date: Wed, 3 Jun 2020 14:55:58 +0100 Subject: [PATCH 057/181] adding icr_davros to test --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0609a85..45fe4bb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 6d356cc9182d6f6ba34c655c9acbc39ba2efbdd0 Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 5 Jun 2020 12:07:26 +0200 Subject: [PATCH 058/181] CBE: don't load singularity module singularity is now installed from CentOS repos, the environment module will be removed --- conf/cbe.config | 1 - docs/cbe.md | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 5888202..0e9b43e 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -7,7 +7,6 @@ params { process { executor = 'slurm' - module = 'singularity/3.4.1' queue = { task.memory <= 170.GB ? 'c' : 'm' } clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } } diff --git a/docs/cbe.md b/docs/cbe.md index 8d597a0..aa6c60d 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -4,13 +4,12 @@ All nf-core pipelines have been successfully configured for use on the CLIP BATC To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on CBE. You can do this by issuing the commands below: +Before running the pipeline you will need to load Nextflow using the environment module system on CBE. You can do this by issuing the commands below: ```bash -## Load Nextflow and Singularity environment modules +## Load Nextflow environment module module purge module load nextflow/19.04.0 -module load singularity/3.2.1 ``` A local copy of the [AWS-iGenomes](https://registry.opendata.aws/aws-igenomes/) resource has been made available on CBE so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. From 17824d2d62a3a548c6bd9ed090ad5ed9b80694e0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 16 Jun 2020 15:20:39 +0200 Subject: [PATCH 059/181] Merge czbiohub_aws_highpriority profile into czbiohub_aws --- README.md | 1 - bin/cchecker.py | 12 ++++++------ conf/czbiohub_aws.config | 9 +++++++++ conf/czbiohub_aws_highpriority.config | 12 ------------ docs/czbiohub.md | 8 ++++++++ nfcore_custom.config | 3 --- 6 files changed, 23 insertions(+), 22 deletions(-) delete mode 100644 conf/czbiohub_aws_highpriority.config diff --git a/README.md b/README.md index e5b199c..7da6090 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,6 @@ Currently documentation is available for the following systems: * [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) -* [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [EBC](docs/ebc.md) * [GENOTOUL](docs/genotoul.md) diff --git a/bin/cchecker.py b/bin/cchecker.py index 4290170..9629cf8 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -45,11 +45,11 @@ def check_config(Config, Github): if re.search(regex, line): hit = line.split('/')[2].split('.')[0] config_profiles.add(hit.strip()) - - ###Check Github Config now + + ### Check Github Config now tests = set() - ###Ignore these profiles - ignore_me = ['czbiohub_aws_highpriority', 'czbiohub_aws'] + ### Ignore these profiles + ignore_me = ['czbiohub_aws'] tests.update(ignore_me) with open(Github, 'r') as ghfile: for line in ghfile: @@ -58,11 +58,11 @@ def check_config(Config, Github): profiles = line.split(':')[1].split(',') for p in profiles: tests.add(p.strip()) - + ###Check if sets are equal if tests == config_profiles: sys.exit(0) - else: + else: #Maybe report what is missing here too print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n") print(config_profiles.symmetric_difference(tests)) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index b32fb6d..c4b0e7d 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -135,3 +135,12 @@ params { } } } + + +profiles { + highpriority { + process { + queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' + } + } +} diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config deleted file mode 100644 index 5ab796a..0000000 --- a/conf/czbiohub_aws_highpriority.config +++ /dev/null @@ -1,12 +0,0 @@ -/* - * ------------------------------------------------- - * Nextflow config file for Chan Zuckerberg Biohub - * ------------------------------------------------- - * Defines reference genomes, using iGenome paths - * Imported under the default 'standard' Nextflow - * profile in nextflow.config - */ - -process { - queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' -} diff --git a/docs/czbiohub.md b/docs/czbiohub.md index 4826670..03f0edd 100644 --- a/docs/czbiohub.md +++ b/docs/czbiohub.md @@ -122,3 +122,11 @@ For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annota >NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. + +## High Priority Queue + +If you would like to run with the _High Priority_ queue, specify the `highpriority` config profile after `czbiohub_aws`. When applied after the main `czbiohub_aws` config, it overwrites the process `queue` identifier. + +To use it, submit your run with with `-profile czbiohub_aws,highpriority`. + +**Note that the order of config profiles here is important.** For example, `-profile highpriority,czbiohub_aws` will not work. diff --git a/nfcore_custom.config b/nfcore_custom.config index 6349a15..aa084dc 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,9 +21,6 @@ profiles { cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { - includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; - includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config"} ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } From dbfcd7c5e91ad3ec2afe225ef59eff6e820f5f78 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 17 Jun 2020 15:17:22 +0200 Subject: [PATCH 060/181] Rename uct_hex > uct_hpc --- conf/{uct_hex.config => uct_hpc.config} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conf/{uct_hex.config => uct_hpc.config} (100%) diff --git a/conf/uct_hex.config b/conf/uct_hpc.config similarity index 100% rename from conf/uct_hex.config rename to conf/uct_hpc.config From 53e118210fbea6a3b6166d73cefa626250d8b303 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 17 Jun 2020 15:20:22 +0200 Subject: [PATCH 061/181] Copy over HPC config --- conf/uct_hpc.config | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/conf/uct_hpc.config b/conf/uct_hpc.config index 6e0bbbe..e7218ba 100644 --- a/conf/uct_hpc.config +++ b/conf/uct_hpc.config @@ -1,23 +1,41 @@ -//Profile config names for nf-core/configs +/* + * ------------------------------------------------- + * HPC cluster config file + * ------------------------------------------------- + * http://www.hpc.uct.ac.za/ + */ + params { - config_profile_description = 'University of Cape Town HEX cluster config file provided by nf-core/configs.' - config_profile_contact = 'Katie Lennard (@kviljoen)' - config_profile_url = 'http://hpc.uct.ac.za/index.php/hex-3/' + config_profile_description = 'University of Cape Town High Performance Cluster config file provided by nf-core/configs.' + config_profile_contact = 'Katie Lennard (@kviljoen)' + config_profile_url = 'http://hpc.uct.ac.za/index.php/hpc-cluster/' + + singularity_cache_dir = "/bb/DB/bio/singularity-containers/" + igenomes_base = '/bb/DB/bio/rna-seq/references' + max_memory = 384.GB + max_cpus = 40 + max_time = 1000.h + hpc_queue = 'ada' + hpc_account = '--account cbio' + genome = 'GRCh37' } singularity { enabled = true - cacheDir = "/scratch/DB/bio/singularity-containers" + cacheDir = params.singularity_cache_dir + autoMounts = true } process { + executor = 'slurm' + queue = params.hpc_queue + // Increasing maxRetries, this will overwrite what we have in base.config + maxRetries = 4 + clusterOptions = params.hpc_account stageInMode = 'symlink' stageOutMode = 'rsync' - queue = 'UCTlong' - clusterOptions = { "-M $params.email -m abe -l nodes=1:ppn=1:series600" } } -executor{ - executor = 'pbs' - jobName = { "$task.tag" } +executor { + queueSize = 15 } From 29f5fa6ce40179e361677b7ea740c99547f14a08 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 17 Jun 2020 15:22:42 +0200 Subject: [PATCH 062/181] uct_hpc docs, readme --- README.md | 2 +- docs/uct_hpc.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 docs/uct_hpc.md diff --git a/README.md b/README.md index e5b199c..4dd4e42 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Currently documentation is available for the following systems: * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) -* [UCT_HEX](docs/uct_hex.md) +* [UCT_HPC](docs/uct_hpc.md) * [UPPMAX](docs/uppmax.md) * [UTD_GANYMEDE](docs/utd_ganymede.md) * [UZH](docs/uzh.md) diff --git a/docs/uct_hpc.md b/docs/uct_hpc.md new file mode 100644 index 0000000..dd24e3f --- /dev/null +++ b/docs/uct_hpc.md @@ -0,0 +1,5 @@ +# nf-core/configs: UCT HPC config + +University of Cape Town [High Performance Cluster](http://hpc.uct.ac.za/index.php/hpc-cluster/) config. + +For help or more information, please contact Katie Lennard (@kviljoen). From 18a1a319ec5db8cfc7e9f05e2fabdc8ce62079e6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 17 Jun 2020 15:25:07 +0200 Subject: [PATCH 063/181] More uct_hex references --- .github/workflows/main.yml | 4 ++-- nfcore_custom.config | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45fe4bb..019a3ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow @@ -24,7 +24,7 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Check ${{ matrix.profile }} profile - env: + env: SCRATCH: '~' NXF_GLOBAL_CONFIG: awsbatch.config run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} diff --git a/nfcore_custom.config b/nfcore_custom.config index 6349a15..f61d0cb 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -38,7 +38,7 @@ profiles { phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } - uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } + uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } From cb26bef3f3bc777bb307f997e0705f6c866c77d1 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Thu, 18 Jun 2020 20:30:05 +0200 Subject: [PATCH 064/181] resolve #162 bug --- conf/bi.config | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index f9dfb27..65059e3 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -3,16 +3,18 @@ params{ config_profile_contact = 'Alexander Peltzer (@apeltzer)' config_profile_url = 'https://www.boehringer-ingelheim.com/' } -params.globalConfig = determine_global_config() -includeConfig params.globalConfig +params.globalConfig = set_global_config() -def determine_global_config() { - if( System.getenv('NXF_GLOBAL_CONFIG') == null) - { - def errorMessage = "ERROR: Environment variable NXF_GLOBAL_CONFIG is missing. Set it to point to global.config file." - System.err.println(errorMessage) - throw new Exception(errorMessage) - } - return System.getenv('NXF_GLOBAL_CONFIG') + +def set_global_config() { + def config = System.getenv('NXF_GLOBAL_CONFIG') + if(config == null) + { + def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile." + System.err.println(errorMessage) + }else{ + includeConfig config + } + return config } From e375ad97ad8dc2b9eddca72cf7d84fcf909c1021 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2020 13:39:37 +0100 Subject: [PATCH 065/181] reverting autoMounts due to bug --- conf/icr_davros.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/icr_davros.config b/conf/icr_davros.config index 9aa572a..841dc2e 100644 --- a/conf/icr_davros.config +++ b/conf/icr_davros.config @@ -13,8 +13,8 @@ params { singularity { enabled = true - // runOptions = "--bind /mnt:/mnt --bind /data:/data" - autoMounts = true + runOptions = "--bind /mnt:/mnt --bind /data:/data" + // autoMounts = true // autoMounts sometimes causes a rare bug with the installed version of singularity } executor { From 5b890d571a1d5c78c809f9248cb48b7364f48d00 Mon Sep 17 00:00:00 2001 From: Adrian Date: Fri, 26 Jun 2020 13:40:15 +0100 Subject: [PATCH 066/181] adding icr_davros to list --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5ac8a6e..170adc1 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ Currently documentation is available for the following systems: * [GIS](docs/gis.md) * [GOOGLE](docs/google.md) * [HEBBE](docs/hebbe.md) +* [ICR_DAVROS](docs/icr_davros.md) * [KRAKEN](docs/kraken.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) From c52428cd72916e232b2f2faaa3d3f17d504b1b98 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 30 Jun 2020 08:19:25 +0200 Subject: [PATCH 067/181] Bump number of eager @shh retries to account for deep sequencing data --- conf/pipeline/eager/shh.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 94a0403..5d8b576 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -13,6 +13,9 @@ params { // Specific nf-core/eager process configuration process { + + maxRetries = 5 + withName: malt { maxRetries = 1 memory = { task.attempt > 1 ? 1900.GB : 725.GB } From 4828e78708ecf6ddb8a2a8db560bb72f72357330 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Thu, 2 Jul 2020 21:57:44 +0200 Subject: [PATCH 068/181] fix error introduced earlier --- conf/bi.config | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index 65059e3..3a77bb7 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -4,17 +4,12 @@ params{ config_profile_url = 'https://www.boehringer-ingelheim.com/' } -params.globalConfig = set_global_config() - - -def set_global_config() { - def config = System.getenv('NXF_GLOBAL_CONFIG') - if(config == null) - { - def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile." - System.err.println(errorMessage) - }else{ - includeConfig config - } - return config +def configVar = System.getenv('NXF_GLOBAL_CONFIG') +if(configVar == null) +{ + def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile." + System.err.println(errorMessage) +}else{ + params.globalConfig = configVar + includeConfig configVar } From 4181184f58892a6c42efba3e9bd1eceaf7a6fd73 Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Thu, 2 Jul 2020 22:00:37 +0200 Subject: [PATCH 069/181] fix spaces --- conf/bi.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index 3a77bb7..fe2ae17 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -10,6 +10,6 @@ if(configVar == null) def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile." System.err.println(errorMessage) }else{ - params.globalConfig = configVar - includeConfig configVar + params.globalConfig = configVar + includeConfig configVar } From 5517d2cbe06f9d30d81c0851341adf90565ddeb8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 7 Jul 2020 13:29:13 +0200 Subject: [PATCH 070/181] Remove default non-profile bwa parameters from eager profile I realised today this is dangerous as these would not necessarily be reported if a user doesn't understand the profiles. Therefore removing these so nf-core/eager defaults are always used unless a specific profile (which would be described in the command itself) is explicitly named. --- conf/pipeline/eager/shh.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 5d8b576..e408f07 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -5,11 +5,7 @@ params { config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" - - // default BWA - bwaalnn = 0.04 - bwaalnl = 32 -} + } // Specific nf-core/eager process configuration process { From cb7a6f756af97154cc2867c484488408caa0babd Mon Sep 17 00:00:00 2001 From: Piotr Faba Date: Tue, 7 Jul 2020 13:48:28 +0200 Subject: [PATCH 071/181] reduce complexity --- conf/bi.config | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/bi.config b/conf/bi.config index fe2ae17..1a218f1 100644 --- a/conf/bi.config +++ b/conf/bi.config @@ -4,12 +4,11 @@ params{ config_profile_url = 'https://www.boehringer-ingelheim.com/' } -def configVar = System.getenv('NXF_GLOBAL_CONFIG') -if(configVar == null) +params.globalConfig = System.getenv('NXF_GLOBAL_CONFIG') +if(params.globalConfig == null) { def errorMessage = "WARNING: For bi.config requires NXF_GLOBAL_CONFIG env var to be set. Point it to global.config file if you want to use this profile." System.err.println(errorMessage) }else{ - params.globalConfig = configVar - includeConfig configVar + includeConfig params.globalConfig } From af01fccba1be7ffa4bc1d84d8e31e4f0a3f6c86f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 7 Jul 2020 14:31:11 +0200 Subject: [PATCH 072/181] Add a HOPS profile for nf-core/eager @ SHH --- conf/pipeline/eager/shh.config | 42 ++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 5d8b576..8001823 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -5,10 +5,6 @@ params { config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs' igenomes_base = "/projects1/public_data/igenomes/" - - // default BWA - bwaalnn = 0.04 - bwaalnl = 32 } // Specific nf-core/eager process configuration @@ -26,6 +22,44 @@ process { } profiles { + hops { + pathogen_screening { + fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' + bwa_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' + fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai' + seq_dict = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.dict' + bwaalnn = 0.04 + bwaalnl = 32 + run_bam_filtering = true + bam_discard_unmapped = true + bam_unmapped_type = 'fastq' + run_metagenomic_screening = true + metagenomic_tool = 'malt' + metagenomic_min_support_reads = 1 + database = '/projects1/malt/databases/indexed/index040/full-bac-full-vir-etal-nov_2017' + percent_identity = 90 + malt_mode = 'BlastN' + malt_alignment_mode = 'SemiGlobal' + malt_top_percent = 1 + malt_min_support_mode = 'percent' + malt_min_support_percent = 0.01 + malt_max_queries = 100 + malt_memory_mode = 'load' + run_maltextract = true + maltextract_taxon_list = '/projects1/users/key/anc5h/soi.backup/List_of_pathogens_KB_fmk12_wViruses1.txt' + maltextract_ncbifiles = '/projects1/clusterhomes/huebler/RMASifter/RMA_Extractor_Resources/' + maltextract_filter = 'def_anc' + maltextract_toppercent = 0.01 + maltextract_destackingoff = false + maltextract_downsamplingoff = false + maltextract_duplicateremovaloff = false + maltextract_matches = false + maltextract_megansummary = false + maltextract_percentidentity = 90.0 + maltextract_topalignment = false + maltextract_singlestranded = false + } + } pathogen_loose { params { config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.' From b2e043ab59e2cf693e6d7e08df49cfbbb93bd960 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 7 Jul 2020 14:35:42 +0200 Subject: [PATCH 073/181] Add documentation for HOPS profile for eager @ SHH --- docs/pipeline/eager/shh.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index 8377ded..b19d935 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -16,3 +16,4 @@ Specific configurations for shh has been made for eager. * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG. * Provides additional group specific profiles, which adapt the `bwa aln` mapping parameters to each context: `pathogens_loose` (`-l 0.01 -n 16`), `pathogens_strict` (`-l 32, -n 0.1`) and `human` (`-l 16500, -n 0.01`). +* Provides a `hops` profile with default paths and parameters for the Pathogen group. From 737d826932ab17efa822af6462c15dec6fc8e05a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 08:36:04 +0200 Subject: [PATCH 074/181] Update shh.config --- conf/pipeline/eager/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 8001823..dabff38 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,7 +23,7 @@ process { profiles { hops { - pathogen_screening { + params { fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' bwa_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai' From aac3478e86f1afd19ce2e99fc34e1c19f2a162c4 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 09:12:11 +0200 Subject: [PATCH 075/181] Add better task time values to account for large data --- conf/pipeline/eager/shh.config | 43 ++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index dabff38..718242c 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -19,6 +19,49 @@ process { time = 1440.h queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + } + } profiles { From 31edb4b5d7b0e5d9f0017251ce992ac982d66aaf Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 09:20:00 +0200 Subject: [PATCH 076/181] Update shh.config --- conf/pipeline/eager/shh.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 718242c..56f5cb9 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } } } From b360ce26bcd5aa69f99722573585655daa88525f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 09:24:06 +0200 Subject: [PATCH 077/181] Update shh.config --- conf/pipeline/eager/shh.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 56f5cb9..935d2a5 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 2.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } } From c6905b609e0422a36501e38b788a5003ded69526 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 09:25:21 +0200 Subject: [PATCH 078/181] Update shh.config --- conf/pipeline/eager/shh.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 935d2a5..3d0f195 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,13 +23,13 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 2.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 2.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_medium'{ From 563c27e1bb053c6658a76c50ab0655c157210fb9 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Wed, 8 Jul 2020 09:36:28 +0200 Subject: [PATCH 079/181] Try again to get better time resource submission --- conf/pipeline/eager/shh.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 3d0f195..718242c 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } } } From 926c14f7ee0368869214671d595179c9777635d1 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Wed, 8 Jul 2020 09:36:56 +0200 Subject: [PATCH 080/181] Cleverer process time submission --- conf/pipeline/eager/shh.config | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 718242c..560dee5 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -10,11 +10,11 @@ params { // Specific nf-core/eager process configuration process { - maxRetries = 5 + maxRetries = 2 withName: malt { maxRetries = 1 - memory = { task.attempt > 1 ? 1900.GB : 725.GB } + memory = { task.attempt > 1 ? 1900.GB : 725.GB } cpus = { task.attempt > 1 ? 112 : 64 } time = 1440.h queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { check_max( task.attempt == 3 ? 120.h : task.attempt == 2 ? 48.h : task.attempt == 1 ? 24.h : 1.h, 'time' ) } + time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 1.h } } } @@ -99,7 +99,7 @@ profiles { maltextract_matches = false maltextract_megansummary = false maltextract_percentidentity = 90.0 - maltextract_topalignment = false + maltextract_topalignment = false maltextract_singlestranded = false } } From 15dad7e2e716869614ce1e2040f3d4e7b633f5c6 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Wed, 8 Jul 2020 09:47:03 +0200 Subject: [PATCH 081/181] Add caveats for EAGER @ SHH profiles --- conf/pipeline/eager/shh.config | 1 + docs/pipeline/eager/shh.md | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 560dee5..9414da2 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -65,6 +65,7 @@ process { } profiles { + // IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY. hops { params { fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index b19d935..f83c37d 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -14,6 +14,19 @@ Example: `nextflow run nf-core/eager -profile shh` Specific configurations for shh has been made for eager. +### General profiles + * If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG. -* Provides additional group specific profiles, which adapt the `bwa aln` mapping parameters to each context: `pathogens_loose` (`-l 0.01 -n 16`), `pathogens_strict` (`-l 32, -n 0.1`) and `human` (`-l 16500, -n 0.01`). -* Provides a `hops` profile with default paths and parameters for the Pathogen group. + +### Contextual profiles + +#### Human Pop-Gen + +* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) + +#### Pathogen + +* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) +* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) +* `hops`: profile with default paths and parameters for automated/initial pathogen screening. + * :warning: This is NOT a reproducible profile as it contains hardcoded paths. This should only be used for initial/automated screening where you wish to quickly check for any possible positives; after which you should re-do screening in a reproducible manner for publication! From c57a404f187f6a51e508f68a302ffecb12d48034 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 10:45:43 +0200 Subject: [PATCH 082/181] Add desc for hops --- conf/pipeline/eager/shh.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 9414da2..ce5e6c4 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -68,6 +68,7 @@ profiles { // IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY. hops { params { + config_profile_description = 'Rough HOPS screening MPI-SHH profile, provided by nf-core/configs.' fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' bwa_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai' From 1af9ce293c0eb38255391e804585f025d1834b04 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 15:22:33 +0200 Subject: [PATCH 083/181] Update hops profile based on requests from pathogen group --- conf/pipeline/eager/shh.config | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index ce5e6c4..ec7bff1 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -73,8 +73,8 @@ profiles { bwa_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta' fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai' seq_dict = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.dict' - bwaalnn = 0.04 - bwaalnl = 32 + bwaalnn = 0.01 + bwaalnl = 16 run_bam_filtering = true bam_discard_unmapped = true bam_unmapped_type = 'fastq' @@ -82,11 +82,11 @@ profiles { metagenomic_tool = 'malt' metagenomic_min_support_reads = 1 database = '/projects1/malt/databases/indexed/index040/full-bac-full-vir-etal-nov_2017' - percent_identity = 90 + percent_identity = 85 malt_mode = 'BlastN' malt_alignment_mode = 'SemiGlobal' malt_top_percent = 1 - malt_min_support_mode = 'percent' + malt_min_support_mode = 'reads' malt_min_support_percent = 0.01 malt_max_queries = 100 malt_memory_mode = 'load' @@ -99,10 +99,9 @@ profiles { maltextract_downsamplingoff = false maltextract_duplicateremovaloff = false maltextract_matches = false - maltextract_megansummary = false - maltextract_percentidentity = 90.0 + maltextract_megansummary = true + maltextract_percentidentity = 85.0 maltextract_topalignment = false - maltextract_singlestranded = false } } pathogen_loose { From 8f6409d1440b7990102d7e5503e1c1c6d38c6c1d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 8 Jul 2020 15:26:19 +0200 Subject: [PATCH 084/181] Remove min support percent value to avoid confusion --- conf/pipeline/eager/shh.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index ec7bff1..b3b6c3e 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -87,7 +87,6 @@ profiles { malt_alignment_mode = 'SemiGlobal' malt_top_percent = 1 malt_min_support_mode = 'reads' - malt_min_support_percent = 0.01 malt_max_queries = 100 malt_memory_mode = 'load' run_maltextract = true From 84e03b8264f1985b01a9cce6a7f2fe899376a4f7 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sat, 11 Jul 2020 18:31:04 +0200 Subject: [PATCH 085/181] Re-bump default walltimes after debugging/testing --- conf/pipeline/eager/shh.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index b3b6c3e..c48db8b 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -23,43 +23,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 1.h } + time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 2.h } } } From 57ac9257a92ef2912163e02425359e2ace8f5aa7 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Mon, 20 Jul 2020 11:29:12 +0200 Subject: [PATCH 086/181] Remove now deprecated EAGER flag for HOPS profile --- conf/pipeline/eager/shh.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index c48db8b..777f516 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -76,7 +76,6 @@ profiles { bwaalnn = 0.01 bwaalnl = 16 run_bam_filtering = true - bam_discard_unmapped = true bam_unmapped_type = 'fastq' run_metagenomic_screening = true metagenomic_tool = 'malt' From 69a0bd572b646db68f9ff1f19cd059d7e59807a2 Mon Sep 17 00:00:00 2001 From: maxibor Date: Thu, 23 Jul 2020 15:38:28 +0200 Subject: [PATCH 087/181] add temporary CDAG setting for SHH config --- conf/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index 8defc1f..e0c7116 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -33,6 +33,10 @@ profiles { params { config_profile_description = 'CDAG MPI-SHH profile, provided by nf-core/configs.' } + // delete when CDAG will be fixed + process { + queue = 'long' + } } sdag { params { From b80407695941bedac700e6a70dd44a305b292eae Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 3 Aug 2020 11:38:10 +0100 Subject: [PATCH 088/181] Remove hard Singularity version --- conf/crick.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/crick.config b/conf/crick.config index 4bcd242..338bb76 100755 --- a/conf/crick.config +++ b/conf/crick.config @@ -11,7 +11,6 @@ singularity { } process { - beforeScript = 'module load Singularity/2.6.0-foss-2016b' executor = 'slurm' } From 165af5122f31f0ff63827366abcf5fc0c67e816a Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 3 Aug 2020 11:39:23 +0100 Subject: [PATCH 089/181] Update docs --- docs/crick.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/crick.md b/docs/crick.md index e18db65..5eae24f 100644 --- a/docs/crick.md +++ b/docs/crick.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load Nextflow/19.10.0 -module load Singularity/2.6.0-foss-2016b +module load Singularity ``` A local copy of the [AWS-iGenomes](https://github.com/ewels/AWS-iGenomes) resource has been made available on CAMP so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. Some of the more exotic genomes may not have been downloaded onto CAMP so have a look in the `igenomes_base` path specified in [`crick.config`](../conf/crick.config), and if your genome of interest isnt present please contact [BABS](mailto:bioinformatics@crick.ac.uk). From c3fa87e4bc4954ab182850837e15c7b8e27fa2b1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Aug 2020 09:18:36 +0200 Subject: [PATCH 090/181] Add microbiome screening profile for NT sized MALT runs for nf-core/eager@SHH --- conf/pipeline/eager/shh.config | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 777f516..b358712 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -65,6 +65,18 @@ process { } profiles { + microbiome_screening { + process { + // Assuming NCBI NT-sized DB + withName: malt { + maxRetries = 1 + memory = 1900.GB + cpus = 112 + time = 1440.h + queue = 'supercruncher' + } + } + } // IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY. hops { params { From 09cfabd37ffffb6f39ce2b6bd0865d40c8dba89d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 11 Aug 2020 09:20:31 +0200 Subject: [PATCH 091/181] Add new microbiome_screening profile to docs --- docs/pipeline/eager/shh.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/pipeline/eager/shh.md b/docs/pipeline/eager/shh.md index f83c37d..c09606d 100644 --- a/docs/pipeline/eager/shh.md +++ b/docs/pipeline/eager/shh.md @@ -20,6 +20,10 @@ Specific configurations for shh has been made for eager. ### Contextual profiles +#### Microbiome Sciences + +* `microbiome_screening` runs MALT straight to supercruncher (with no retries!) and full resources requested due to microbiome screening databases often easily reach this size + #### Human Pop-Gen * `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) From 42a1a159700c0ca5b551ccd97a9126ad0f767f2e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:11:56 +0200 Subject: [PATCH 092/181] Create mpcdf.config --- conf/mpcdf.config | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 conf/mpcdf.config diff --git a/conf/mpcdf.config b/conf/mpcdf.config new file mode 100644 index 0000000..9a109b0 --- /dev/null +++ b/conf/mpcdf.config @@ -0,0 +1,30 @@ +params { + config_profile_description = 'MPCDF HPC profiles provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://www.mpcdf.mpg.de/services/computing' +} + + +profiles { + cobra { + // Does not have singularity! Conda module must be used, but it is + // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} + // To create common cache dir (only until /u/ periodically cleared!) + + process { + beforeScript = 'module load anaconda/3/2020.02' + executor = 'slurm' + } + + executor { + queueSize =8 + } + + params { + config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' + max_memory = 750.GB + max_cpus = 80 + max_time = 24.h + } + } +} From 0501ae4c1a1f7c7d3421d3bf7075de75d9f3bfce Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:21:49 +0200 Subject: [PATCH 093/181] Started docs for mpcdf --- docs/mpcdf.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/mpcdf.md diff --git a/docs/mpcdf.md b/docs/mpcdf.md new file mode 100644 index 0000000..11d4474 --- /dev/null +++ b/docs/mpcdf.md @@ -0,0 +1,22 @@ +# nf-core/configs: MPCDF Configuration + +All nf-core pipelines have been successfully configured for use on the HPCs at [Max Planck Computing and Data Facility](https://www.mpcdf.mpg.de/). + +> :warning: these profiles are not officially supported by the MPCDF. + +To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. + +Currently the following clusters are supported: cobra, draco* (* coming soon) + +>NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). + +# cobra + +Cobra does not currently support singularity, therefore the anaconda/module should be loaded for use. + +Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). + +To use: `-profile cobra,mpcdf` + +# draco + From a1c5dc5fde9e3877166488dcdb9a0e2eb4e91a07 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:22:27 +0200 Subject: [PATCH 094/181] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 170adc1..1229c7e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Currently documentation is available for the following systems: * [HEBBE](docs/hebbe.md) * [ICR_DAVROS](docs/icr_davros.md) * [KRAKEN](docs/kraken.md) +* [MPCDF](docs/mpcdf.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) From 0a685c1cc2568fe06962e180e4c31f48defb7efa Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:24:11 +0200 Subject: [PATCH 095/181] Update mpcdf.config --- conf/mpcdf.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 9a109b0..849ba9c 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -1,5 +1,5 @@ params { - config_profile_description = 'MPCDF HPC profiles provided by nf-core/configs.' + config_profile_description = 'MPCDF HPC profiles (unoffically) provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://www.mpcdf.mpg.de/services/computing' } @@ -17,7 +17,7 @@ profiles { } executor { - queueSize =8 + queueSize = 8 } params { From 09cd5cad6565e99cc872064d63a93e8362ffd35b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:25:37 +0200 Subject: [PATCH 096/181] Update mpcdf.md --- docs/mpcdf.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 11d4474..89712d0 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -12,11 +12,18 @@ Currently the following clusters are supported: cobra, draco* (* coming soon) # cobra -Cobra does not currently support singularity, therefore the anaconda/module should be loaded for use. +Cobra does not currently support singularity, therefore the anaconda/module is loaded for each process. Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). To use: `-profile cobra,mpcdf` +Sets the following parameters: + +Maximum parallel running jobs: 8 +Max. memory: 750.GB +Max. CPUs: 80 +Max. walltime: 24.h + # draco From 9df85be081f386c19e303d1529df39aa816053f1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:27:15 +0200 Subject: [PATCH 097/181] Update mpcdf.md --- docs/mpcdf.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 89712d0..876dd8b 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -6,11 +6,11 @@ All nf-core pipelines have been successfully configured for use on the HPCs at [ To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. -Currently the following clusters are supported: cobra, draco* (* coming soon) +Currently the following clusters are supported: cobra, draco\* (\* coming soon) >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). -# cobra +## cobra Cobra does not currently support singularity, therefore the anaconda/module is loaded for each process. @@ -25,5 +25,4 @@ Max. memory: 750.GB Max. CPUs: 80 Max. walltime: 24.h -# draco - +## draco From 65fce5c2a539df834809eaf11d444bc545156694 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:52:46 +0200 Subject: [PATCH 098/181] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index c856521..cebbd52 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,6 +30,7 @@ profiles { gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } + mpcdf { includeConfig "${params.custom_config_base}/conf/mpcdf.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } From 5c7ab234b6d4d758dffbe435f6f972f94ed1c028 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:55:40 +0200 Subject: [PATCH 099/181] Create mpcdf.config --- conf/pipeline/eager/mpcdf.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 conf/pipeline/eager/mpcdf.config diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/conf/pipeline/eager/mpcdf.config @@ -0,0 +1 @@ + From b4e275c3b99db0dd3d0b3fcacba605bd72d48939 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:57:25 +0200 Subject: [PATCH 100/181] Add MPCDF for EAGER --- conf/pipeline/eager/mpcdf.config | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 8b13789..5c26a7a 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -1 +1,61 @@ +// Profile config names for nf-core/configs +params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager MPCDF profile provided by nf-core/configs' +} + +// Specific nf-core/eager process configuration +process { + + withName: malt { + maxRetries = 1 + memory = 750.gb + cpus = 80 + time = 24.h + } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = 24.h + } + +} From 4f991d2c0c47574fd7755e617c41571850ba1485 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:59:27 +0200 Subject: [PATCH 101/181] Update eager.config --- pipeline/eager.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/eager.config b/pipeline/eager.config index 36f7cae..9242ecb 100644 --- a/pipeline/eager.config +++ b/pipeline/eager.config @@ -10,4 +10,6 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/pipeline/eager/shh.config" } -} \ No newline at end of file + mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } + +} From af8d25ed0922fc3b921a5e26205b57d86529856e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 20:14:35 +0200 Subject: [PATCH 102/181] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 019a3ee..2d3f9c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 4335e5deb7c2d4a38858ed15f393a2af48b6516e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 20:17:09 +0200 Subject: [PATCH 103/181] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 5c26a7a..ead449d 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -6,6 +6,8 @@ params { config_profile_description = 'nf-core/eager MPCDF profile provided by nf-core/configs' } +profile { +cobra { // Specific nf-core/eager process configuration process { @@ -57,5 +59,6 @@ process { memory = { check_max( 256.GB * task.attempt, 'memory' ) } time = 24.h } - + } + } } From 3b7902d11b29ef95de8877001104153229377d8a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 6 Sep 2020 08:23:35 +0200 Subject: [PATCH 104/181] Mad multi-core increase CPUs on retry --- conf/pipeline/eager/mpcdf.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index ead449d..832e5ed 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -37,25 +37,25 @@ process { } withLabel:'mc_small'{ - cpus = { check_max( 2, 'cpus' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_medium' { - cpus = { check_max( 4, 'cpus' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_huge'{ - cpus = { check_max( 32, 'cpus' ) } + cpus = { check_max( 32 * task.attempt, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } time = 24.h } From dc2735207e297a9bf9a9208d8b551b3a9f0f6eb4 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 13 Sep 2020 20:26:45 -0500 Subject: [PATCH 105/181] feat(utd): add Kim queue --- conf/utd_ganymede.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 465deb5..ce975bd 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -12,13 +12,13 @@ singularity { } process { - beforeScript = 'module load singularity/2.4.5' + beforeScript = 'module load singularity/3.2.1' executor = 'slurm' - queue = 'genomics' + queue = { task.memory >= 32.GB ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' } } params { - max_memory = 32.GB + max_memory = 128.GB max_cpus = 16 - max_time = 48.h + max_time = 96.h } From 2c06019d896e69642d8278d448af8c32deceb8fc Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 13 Sep 2020 20:45:34 -0500 Subject: [PATCH 106/181] fix(utd): Add check for less than 12 cpu --- conf/utd_ganymede.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index ce975bd..409dc0a 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -14,7 +14,7 @@ singularity { process { beforeScript = 'module load singularity/3.2.1' executor = 'slurm' - queue = { task.memory >= 32.GB ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' } + queue = { task.memory >= 32.GB && task.cpu <= 12 ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' } } params { From bf87ec24e7bfe6b654b7060cb33c71e5d37917b8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 17:45:38 +0200 Subject: [PATCH 107/181] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 832e5ed..046b3dc 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -13,7 +13,7 @@ process { withName: malt { maxRetries = 1 - memory = 750.gb + memory = 750.GB cpus = 80 time = 24.h } From c6f7573d01c138ada32e0efa2bccbb5082d74810 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 17:48:54 +0200 Subject: [PATCH 108/181] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 046b3dc..796bad9 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -13,7 +13,7 @@ process { withName: malt { maxRetries = 1 - memory = 750.GB + memory = 725.GB cpus = 80 time = 24.h } From 3115963b3e0575e90136beb674c68eb4fa465c82 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 19:53:27 +0200 Subject: [PATCH 109/181] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 849ba9c..dafb362 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -22,7 +22,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' - max_memory = 750.GB + max_memory = 725.GB max_cpus = 80 max_time = 24.h } From 5b3492d24064535eef70e05d75a7643c494950f3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:26 +0200 Subject: [PATCH 110/181] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index dafb362..7d51680 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -23,7 +23,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' max_memory = 725.GB - max_cpus = 80 + max_cpus = 40 max_time = 24.h } } From bbbdd3cb0fc8b97321b012453ac5cf8124c213ef Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:42 +0200 Subject: [PATCH 111/181] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 7d51680..dafb362 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -23,7 +23,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' max_memory = 725.GB - max_cpus = 40 + max_cpus = 80 max_time = 24.h } } From 11f1fb71db7bffbfe7740c4b9bcc1c2dbad5f6df Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:58 +0200 Subject: [PATCH 112/181] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 796bad9..1b08e73 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -14,7 +14,7 @@ process { withName: malt { maxRetries = 1 memory = 725.GB - cpus = 80 + cpus = 40 time = 24.h } From fbe7bd133ab06a7d0ce0ecc28be4688586e8aaea Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 25 Sep 2020 08:33:53 +0200 Subject: [PATCH 113/181] update uppmax specific sarek config --- conf/pipeline/sarek/uppmax.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index df2f3a2..c5aa2b0 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -4,18 +4,18 @@ params { config_profile_contact = 'Maxime Garcia (@MaxUlysse)' config_profile_description = 'nf-core/sarek uppmax profile provided by nf-core/configs' - singleCPUmem = 7000.MB + single_cpu_mem = 7000.MB // Just useful until iGenomes is updated on UPPMAX - igenomeIgnore = true + igenomes_ignore = true genomes_base = params.genome == 'GRCh37' ? '/sw/data/uppnex/ToolBox/ReferenceAssemblies/hg38make/bundle/2.8/b37' : '/sw/data/uppnex/ToolBox/hg38bundle' } def hostname = "hostname".execute().text.trim() if (hostname ==~ "r.*") { - params.singleCPUmem = 6400.MB + params.single_cpu_mem = 6400.MB } if (hostname ==~ "i.*") { - params.singleCPUmem = 15.GB + params.single_cpu_mem = 15.GB } From 81f910cc95d2f9eb2d93418c6e12ffc7b46539d4 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 25 Sep 2020 09:49:43 +0200 Subject: [PATCH 114/181] add specific BamQC config --- conf/pipeline/sarek/uppmax.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index c5aa2b0..694515c 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -14,6 +14,13 @@ def hostname = "hostname".execute().text.trim() if (hostname ==~ "r.*") { params.single_cpu_mem = 6400.MB + process { + withName:BamQC { + cpus = {params.max_cpus} + memory = {params.max_memory} + } + } + } if (hostname ==~ "i.*") { From 8c0cc4b9ec2e2d24fc4b11a63f2303a3e5277786 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 25 Sep 2020 09:57:50 +0200 Subject: [PATCH 115/181] code polishing --- conf/pipeline/sarek/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/uppmax.config b/conf/pipeline/sarek/uppmax.config index 694515c..46d95a5 100644 --- a/conf/pipeline/sarek/uppmax.config +++ b/conf/pipeline/sarek/uppmax.config @@ -14,13 +14,13 @@ def hostname = "hostname".execute().text.trim() if (hostname ==~ "r.*") { params.single_cpu_mem = 6400.MB + process { withName:BamQC { cpus = {params.max_cpus} memory = {params.max_memory} } } - } if (hostname ==~ "i.*") { From 2ccbdd3367d62866ca25720984eb51eebb19bde8 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Mon, 28 Sep 2020 13:59:42 +0200 Subject: [PATCH 116/181] update params to latest version --- conf/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index 037070a..bc3d8c4 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -15,7 +15,7 @@ process { } params { - saveReference = true + save_reference = true max_memory = 125.GB max_cpus = 16 From e26c22b2f1a6580ab615a53a7619da72326a3af8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 29 Sep 2020 14:03:35 +0200 Subject: [PATCH 117/181] Update shh.config --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index e0c7116..a4d9982 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -25,7 +25,7 @@ process { } executor { - queueSize = 16 + queueSize = 8 } profiles { From 05deeca2d6e5cb4f2e9bfadf4ddf23b0bafca829 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Oct 2020 14:22:35 +0200 Subject: [PATCH 118/181] Update mpcdf.config --- conf/mpcdf.config | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index dafb362..8a0eed2 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -9,7 +9,7 @@ profiles { cobra { // Does not have singularity! Conda module must be used, but it is // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} - // To create common cache dir (only until /u/ periodically cleared!) + // To create common cache dir process { beforeScript = 'module load anaconda/3/2020.02' @@ -21,10 +21,33 @@ profiles { } params { - config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' + config_profile_description = 'MPCDF cobra profile (unofficially) provided by nf-core/configs.' max_memory = 725.GB max_cpus = 80 max_time = 24.h } } + + raven { + // Does not have singularity! Conda module must be used, but it is + // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} + // to create common cache dir + + process { + beforeScript = 'module load anaconda/3/2020.02' + executor = 'slurm' + } + + executor { + queueSize = 8 + } + + params { + config_profile_description = 'MPCDF raven profile (unofficially) provided by nf-core/configs.' + max_memory = 368.GB + max_cpus = 192 + max_time = 24.h + } + } + } From 0d6e269322bd4de0e0d2cdc39346023e9bfb6a41 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Oct 2020 14:24:08 +0200 Subject: [PATCH 119/181] Update mpcdf.md --- docs/mpcdf.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 876dd8b..52f0a71 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -26,3 +26,20 @@ Max. CPUs: 80 Max. walltime: 24.h ## draco + +:hammer_and_wrench: under testing. + +## raven + +Raven does not currently support singularity, therefore the anaconda/module is loaded for each process. + +Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). + +To use: `-profile raven,mpcdf` + +Sets the following parameters: + +Maximum parallel running jobs: 8 +Max. memory: 368.GB +Max. CPUs: 192 +Max. walltime: 24.h From 3995057d7f8e83c454e25b0e535cc22e45fa1ee6 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Oct 2020 10:42:46 +0200 Subject: [PATCH 120/181] Update mpcdf.config --- conf/mpcdf.config | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 8a0eed2..481ebc4 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -27,8 +27,7 @@ profiles { max_time = 24.h } } - - raven { + raven { // Does not have singularity! Conda module must be used, but it is // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} // to create common cache dir @@ -49,5 +48,4 @@ profiles { max_time = 24.h } } - -} +} From 7088ed196fac71055ce9504c6ec79a05cef6ad94 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Oct 2020 11:08:29 +0200 Subject: [PATCH 121/181] Update mpcdf.md --- docs/mpcdf.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 52f0a71..af16d76 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -6,7 +6,7 @@ All nf-core pipelines have been successfully configured for use on the HPCs at [ To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. -Currently the following clusters are supported: cobra, draco\* (\* coming soon) +Currently the following clusters are supported: cobra, raven >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). @@ -20,10 +20,10 @@ To use: `-profile cobra,mpcdf` Sets the following parameters: -Maximum parallel running jobs: 8 -Max. memory: 750.GB -Max. CPUs: 80 -Max. walltime: 24.h +- Maximum parallel running jobs: 8 +- Max. memory: 750.GB +- Max. CPUs: 80 +- Max. walltime: 24.h ## draco @@ -31,7 +31,7 @@ Max. walltime: 24.h ## raven -Raven does not currently support singularity, therefore the anaconda/module is loaded for each process. +Raven does not currently support singularity, therefore `module load anaconda/3/2020.02` is loaded for each process. Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). @@ -39,7 +39,7 @@ To use: `-profile raven,mpcdf` Sets the following parameters: -Maximum parallel running jobs: 8 -Max. memory: 368.GB -Max. CPUs: 192 -Max. walltime: 24.h +- Maximum parallel running jobs: 8 +- Max. memory: 368.GB +- Max. CPUs: 192 +- Max. walltime: 24.h From a63ca751ee74778de4c22985f1cf44ec33e1d197 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 16:57:41 +0200 Subject: [PATCH 122/181] Add ABiMS --- conf/abims.config | 24 ++++++++++++++++++++++++ docs/abims.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 conf/abims.config create mode 100644 docs/abims.md diff --git a/conf/abims.config b/conf/abims.config new file mode 100644 index 0000000..47be8cd --- /dev/null +++ b/conf/abims.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The ABiMS cluster profile' + config_profile_contact = 'Gildas Le Corguillé (@lecorguille)' + config_profile_url = 'https://abims.sb-roscoff.fr' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /scratch:/scratch -B /scratch2:/scratch2 -B /shared:/shared' +} + +process { + executor = 'slurm' +} + +params { + igenomes_ignore = true + igenomesIgnore = true //deprecated + max_memory = 750.GB + max_cpus = 200 + max_time = 24.h +} diff --git a/docs/abims.md b/docs/abims.md new file mode 100644 index 0000000..a3298e5 --- /dev/null +++ b/docs/abims.md @@ -0,0 +1,46 @@ +# nf-core/configs: ABiMS Configuration + +All nf-core pipelines have been successfully configured for use on the ABiMS cluster. + +To use, run the pipeline with `-profile abims`. This will download and launch the [`abims.config`](../conf/abims.config) which has been pre-configured with a setup suitable for the ABiMS cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Request an account +You will need an account to use the HPC cluster on ABiMS in order +to run the pipeline. If in doubt see [http://abims.sb-roscoff.fr/account](http://abims.sb-roscoff.fr/account). + +## Running the workflow on the ABiMS cluster + +Nextflow is installed on the ABiMS cluster. Some documentation is available on the [ABiMS website](http://abims.sb-roscoff.fr/resources/cluster/howto#nextflow). + +You need to activate it like this: + +```bash +module load nextflow/20.04.1 slurm-drmaa/1.0.8 +``` + +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. Run nextflow from a compute node: + +```bash +# Login to a compute node +srun --pty bash + +# Load the dependencies if not done before +module load nextflow/20.04.1 slurm-drmaa/1.0.8 + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +/path/to/nf-core/workflow \\ +-resume +-profile abims \\ +--email my-email@example.org \\ +-c my-specific.config +... + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... +``` + +## Databanks + +A local copy of several genomes are available in `/shared/bank` directory. From f21d16b24f862905a026db04405dfc5caff89bc7 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 17:47:25 +0200 Subject: [PATCH 123/181] Add ABiMS - fix lint --- docs/abims.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/abims.md b/docs/abims.md index a3298e5..43896ab 100644 --- a/docs/abims.md +++ b/docs/abims.md @@ -5,6 +5,7 @@ All nf-core pipelines have been successfully configured for use on the ABiMS clu To use, run the pipeline with `-profile abims`. This will download and launch the [`abims.config`](../conf/abims.config) which has been pre-configured with a setup suitable for the ABiMS cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. ## Request an account + You will need an account to use the HPC cluster on ABiMS in order to run the pipeline. If in doubt see [http://abims.sb-roscoff.fr/account](http://abims.sb-roscoff.fr/account). From 90d1f9072bd04b9144d2260c7d1f86c1a412ce54 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 18:18:55 +0200 Subject: [PATCH 124/181] Add ABiMS - autoMounts = false --- conf/abims.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/abims.config b/conf/abims.config index 47be8cd..79c73d2 100644 --- a/conf/abims.config +++ b/conf/abims.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - autoMounts = true + autoMounts = false runOptions = '-B /scratch:/scratch -B /scratch2:/scratch2 -B /shared:/shared' } From 23ba1c58b1aaf0c5f3e38533d3b8a00de26c4546 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 23:17:55 +0200 Subject: [PATCH 125/181] Add ABiMS - NXF_SINGULARITY_CACHEDIR --- docs/abims.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/abims.md b/docs/abims.md index 43896ab..87cc5c4 100644 --- a/docs/abims.md +++ b/docs/abims.md @@ -16,7 +16,7 @@ Nextflow is installed on the ABiMS cluster. Some documentation is available on t You need to activate it like this: ```bash -module load nextflow/20.04.1 slurm-drmaa/1.0.8 +module load nextflow slurm-drmaa graphviz ``` Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. @@ -27,7 +27,7 @@ Nextflow shouldn't run directly on the submission node but on a compute node. Ru srun --pty bash # Load the dependencies if not done before -module load nextflow/20.04.1 slurm-drmaa/1.0.8 +module load nextflow slurm-drmaa graphviz # Run a downloaded/git-cloned nextflow workflow from nextflow run \\ @@ -42,6 +42,12 @@ nextflow run \\ nextflow run nf-core/rnaseq ... ``` +## Singularity images mutualized directory + +To reduce the disk usage, nf-core images can be stored in a mutualized directory: `/shared/software/singularity/images/nf-core/` + +The environment variable `NXF_SINGULARITY_CACHEDIR: /shared/data/cache/nextflow` will indicate this directory to nextflow. + ## Databanks -A local copy of several genomes are available in `/shared/bank` directory. +A local copy of several genomes are available in `/shared/bank/` directory. From c9749725855a601946d34c8ff80e93d8f14b253e Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 23:23:56 +0200 Subject: [PATCH 126/181] Add ABiMS - side files --- .github/workflows/main.yml | 2 +- README.md | 1 + nfcore_custom.config | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2d3f9c4..dfb0782 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 1229c7e..a967049 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,7 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following systems: +* [ABIMS](docs/abims.md) * [AWSBATCH](docs/awsbatch.md) * [BIGPURPLE](docs/bigpurple.md) * [BI](docs/bi.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index cebbd52..2b3c419 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -10,6 +10,7 @@ //Please use a new line per include Config section to allow easier linting/parsing. Thank you. profiles { + abims { includeConfig "${params.custom_config_base}/conf/abims.config" } awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bi { includeConfig "${params.custom_config_base}/conf/bi.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } From 6ee6b7bf566ac5260c42504c8c2a7c7307bf624f Mon Sep 17 00:00:00 2001 From: lecorguille Date: Thu, 22 Oct 2020 23:36:33 +0200 Subject: [PATCH 127/181] Add ABiMS - remove a useless link --- docs/abims.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/abims.md b/docs/abims.md index 87cc5c4..2628abb 100644 --- a/docs/abims.md +++ b/docs/abims.md @@ -11,7 +11,7 @@ to run the pipeline. If in doubt see [http://abims.sb-roscoff.fr/account](http:/ ## Running the workflow on the ABiMS cluster -Nextflow is installed on the ABiMS cluster. Some documentation is available on the [ABiMS website](http://abims.sb-roscoff.fr/resources/cluster/howto#nextflow). +Nextflow is installed on the ABiMS cluster. You need to activate it like this: From 4d03f72c78443248f91fe2e8809d90f4b18fd6c1 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Fri, 23 Oct 2020 09:27:12 +0200 Subject: [PATCH 128/181] Add ABiMS - last bits of doc --- docs/abims.md | 47 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/docs/abims.md b/docs/abims.md index 2628abb..c25c80f 100644 --- a/docs/abims.md +++ b/docs/abims.md @@ -23,23 +23,50 @@ Nextflow manages each process as a separate job that is submitted to the cluster Nextflow shouldn't run directly on the submission node but on a compute node. Run nextflow from a compute node: ```bash -# Login to a compute node -srun --pty bash - # Load the dependencies if not done before module load nextflow slurm-drmaa graphviz # Run a downloaded/git-cloned nextflow workflow from -nextflow run \\ -/path/to/nf-core/workflow \\ --resume --profile abims \\ ---email my-email@example.org \\ +srun nextflow run \ +/path/to/nf-core/workflow \ +-profile abims \ +--email my-email@example.org \ -c my-specific.config ... -# Or use the nf-core client -nextflow run nf-core/rnaseq ... +# Or use let nf-core client download the workflow +srun nextflow run nf-core/rnaseq -profile abims ... + +# To launch in background +sbatch --wrap "nextflow run nf-core/rnaseq -profile abims ..." +``` + +Or write a sbatch script + +> nfcore-rnaseq.sh + +```bash +#!/bin/bash +#SBATCH -p fast +#SBATCH --mem=4G + +module load nextflow slurm-drmaa graphviz +nextflow run nf-core/rnaseq -profile abims ... +``` + +Launch on the cluster with sbatch: + +```bash +sbatch nfcore-rnaseq.sh +``` + +### Hello, world! + +nf-core provides some test for each workflow: + +```bash +module load nextflow slurm-drmaa graphviz +nextflow run nf-core/rnaseq -profile abims,test ``` ## Singularity images mutualized directory From e3d733d87ef53dd66a99ac59882ea87bf1f47648 Mon Sep 17 00:00:00 2001 From: lecorguille Date: Fri, 23 Oct 2020 09:29:54 +0200 Subject: [PATCH 129/181] Add ABiMS - fix lint --- docs/abims.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/abims.md b/docs/abims.md index c25c80f..1d1009c 100644 --- a/docs/abims.md +++ b/docs/abims.md @@ -60,7 +60,7 @@ Launch on the cluster with sbatch: sbatch nfcore-rnaseq.sh ``` -### Hello, world! +### Hello, world nf-core provides some test for each workflow: From 9d4d682867d67cf98f5847a7d676b82cdf9284a3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 10 Nov 2020 15:14:03 +0100 Subject: [PATCH 130/181] Update shh.config --- conf/shh.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index e0c7116..0c89be4 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -15,7 +15,6 @@ cleanup = true singularity { enabled = true autoMounts = true - runOptions = '-B /run/shm:/run/shm' cacheDir = "/projects1/singularity_scratch/cache/" } From d70fc2605ac1dafb916d1999c95c46c9f89738a9 Mon Sep 17 00:00:00 2001 From: Andrew Frank Date: Thu, 12 Nov 2020 11:52:58 -0500 Subject: [PATCH 131/181] Fix parameters in google.md Change parameters from hyphens to underscores to accurately reflect google.config --- docs/google.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/google.md b/docs/google.md index 9e8c521..d488b8a 100644 --- a/docs/google.md +++ b/docs/google.md @@ -14,21 +14,21 @@ nextflow run nf-core/rnaseq -profile test,google --google_bucket Date: Tue, 17 Nov 2020 19:45:06 +0100 Subject: [PATCH 132/181] Move sdag queue specification block in sdag profile --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 02c186b..7f2fb8c 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -20,7 +20,6 @@ singularity { process { executor = 'slurm' - queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } executor { @@ -42,6 +41,7 @@ profiles { config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 + queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } } } From 71d1ff2226d7393d65f81cb9eec569d39291e7af Mon Sep 17 00:00:00 2001 From: "Thiseas C. Lamnidis" Date: Wed, 18 Nov 2020 13:04:46 +0100 Subject: [PATCH 133/181] Add params block to sdag queue Co-authored-by: James A. Fellows Yates --- conf/shh.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 7f2fb8c..dc51e87 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -38,10 +38,12 @@ profiles { } sdag { params { - config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' - max_memory = 2.TB - max_cpus = 128 - queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } + config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' + max_memory = 2.TB + max_cpus = 128 + } + process { + queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } } } From 2ea06ffab834d82bd2e9a05dd46c9b7863879bd1 Mon Sep 17 00:00:00 2001 From: Emelie Nilsso Date: Mon, 23 Nov 2020 21:20:36 +0100 Subject: [PATCH 134/181] Modified uppmax and ampliseq specific config to fit with the divided preparation of the database --- conf/pipeline/ampliseq/uppmax.config | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config index 36a1c3b..c594798 100644 --- a/conf/pipeline/ampliseq/uppmax.config +++ b/conf/pipeline/ampliseq/uppmax.config @@ -1,15 +1,27 @@ // Profile config names for nf-core/configs - +System.err.println("reading from the extra uppmax config, e.g. Daniels") params { // Specific nf-core/configs params config_profile_contact = 'Daniel Lundin (daniel.lundin@lnu.se)' config_profile_description = 'nf-core/ampliseq UPPMAX profile provided by nf-core/configs' } -withName: make_SILVA_132_16S_classifier { - clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } -} +process { + //"make_classifier" is the older version (just one step) of preparing the database for classification + withName: make_classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } + } -withName: classifier { - clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } + //"clfr_extract_seq" and "clfr_train" is the newer version, where database preparation is split in two to optimise resources + withName: clfr_extract_seq { + clusterOptions = { "-A $params.project -p core -n 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } + } + + withName: clfr_train { + clusterOptions = { "-A $params.project -C fat -p node -N 1 -t 24:00:00 ${params.clusterOptions ?: ''}" } + } + + withName: classifier { + clusterOptions = { "-A $params.project -C fat -p node -N 1 ${params.clusterOptions ?: ''}" } + } } From 37e7cfae2c92cf87c2e8e0828ad1e07e093ebe0e Mon Sep 17 00:00:00 2001 From: Emelie Nilsso Date: Tue, 24 Nov 2020 07:14:23 +0100 Subject: [PATCH 135/181] Removed unnecessary code that was used to troubleshoot --- conf/pipeline/ampliseq/uppmax.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config index c594798..9f618fb 100644 --- a/conf/pipeline/ampliseq/uppmax.config +++ b/conf/pipeline/ampliseq/uppmax.config @@ -1,5 +1,4 @@ // Profile config names for nf-core/configs -System.err.println("reading from the extra uppmax config, e.g. Daniels") params { // Specific nf-core/configs params config_profile_contact = 'Daniel Lundin (daniel.lundin@lnu.se)' From d61ed01d2dde116b0ea0e068fac210f2d80c1a3f Mon Sep 17 00:00:00 2001 From: Emelie Nilsso Date: Wed, 25 Nov 2020 17:50:30 +0100 Subject: [PATCH 136/181] Updated process names according to ampliseq main.nf --- conf/pipeline/ampliseq/uppmax.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config index 9f618fb..b6cfea2 100644 --- a/conf/pipeline/ampliseq/uppmax.config +++ b/conf/pipeline/ampliseq/uppmax.config @@ -11,12 +11,12 @@ process { clusterOptions = { "-A $params.project -C fat -p node -N 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } } - //"clfr_extract_seq" and "clfr_train" is the newer version, where database preparation is split in two to optimise resources - withName: clfr_extract_seq { + //"classifier_extract_seq" and "classifier_train" is the newer version, where database preparation is split in two to optimise resources + withName: classifier_extract_seq { clusterOptions = { "-A $params.project -p core -n 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } } - withName: clfr_train { + withName: classifier_train { clusterOptions = { "-A $params.project -C fat -p node -N 1 -t 24:00:00 ${params.clusterOptions ?: ''}" } } From d9a800966213d301dd4f2c402169d4db0faf40e7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 25 Nov 2020 22:45:16 +0100 Subject: [PATCH 137/181] Better UPPMAX docs --- docs/uppmax.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 0f23cd1..8551067 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -2,6 +2,10 @@ All nf-core pipelines have been successfully configured for use on the Swedish UPPMAX clusters. +## Getting help + +We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https://nfcore.slack.com/channels/uppmax](https://nfcore.slack.com/channels/uppmax) + ## Using the UPPMAX config profile To use, run the pipeline with `-profile uppmax` (one hyphen). @@ -12,14 +16,19 @@ In addition to this config profile, you will also need to specify an UPPMAX proj You can do this with the `--project` flag (two hyphens) when launching nextflow. For example: ```bash -nextflow run nf-core/PIPELINE -profile uppmax --project SNIC 2018/1-234 # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile uppmax --project "snic2018-1-234" # ..rest of pipeline flags ``` +> NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. + Before running the pipeline you will need to either install Nextflow or load it using the environment module system. -This config enables Nextflow to manage the pipeline jobs via the Slurm job scheduler. +This config enables Nextflow to manage the pipeline jobs via the Slurm job scheduler and using Singularity for software management. + Just run Nextflow on a login node and it will handle everything else. +Remember to use `-bg` to launch Nextflow in the background, so that the pipeline doesn't exit if you leave your terminal session. + ## Using iGenomes references A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `igenomes.config`. @@ -40,7 +49,7 @@ Note that each job will still start with the same request as normal, but restart All jobs will be submitted to fat nodes using this method, so it's only for use in extreme circumstances. -## How to specify a UPPMAX cluster +## Different UPPMAX clusters The UPPMAX nf-core configuration profile uses the `hostname` of the active environment to automatically apply the following resource limits: From e96455e1809cdb0221fbbec502b933b8664978c1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 26 Nov 2020 01:43:09 +0100 Subject: [PATCH 138/181] Update docs/uppmax.md --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 8551067..dcd6ca0 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -16,7 +16,7 @@ In addition to this config profile, you will also need to specify an UPPMAX proj You can do this with the `--project` flag (two hyphens) when launching nextflow. For example: ```bash -nextflow run nf-core/PIPELINE -profile uppmax --project "snic2018-1-234" # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile uppmax --project snic2018-1-234 # ..rest of pipeline flags ``` > NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. From 06469901fe1aac666878353cfc848fd5b5e54f4e Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 11:24:29 +0100 Subject: [PATCH 139/181] typo --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index e7d5327..170d618 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -82,7 +82,7 @@ For security reasons, there is no internet access on Bianca so you can't downloa You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use) Note that you will have to download the singularity images as well. -After transfering the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : +After transffering the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : `export NXF_SINGULARITY_CACHEDIR=Your_Location_For_The_Singularity_directory/.` From 5726ef3456210e4244aeb01a5eed776dbddadf89 Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 11:26:08 +0100 Subject: [PATCH 140/181] typo --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 170d618..1823408 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -79,7 +79,7 @@ To use it, submit with `-profile uppmax,devel`. For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your bianca project. -You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use) Note that you will have to download the singularity images as well. +You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. After transffering the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : From e6bb7a48521da14b2ca783e9af56fc655aa4a670 Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 11:37:59 +0100 Subject: [PATCH 141/181] capital names --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 1823408..24dcf20 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -77,7 +77,7 @@ To use it, submit with `-profile uppmax,devel`. ## Running on Bianca -For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your bianca project. +For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on Bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your Bianca project. You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. From f84fbb8f890d87ef25b3197ca5e4bdcf8adae4cd Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 11:39:51 +0100 Subject: [PATCH 142/181] typo --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 24dcf20..680feab 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -82,7 +82,7 @@ For security reasons, there is no internet access on Bianca so you can't downloa You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. -After transffering the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : +After transferring the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : `export NXF_SINGULARITY_CACHEDIR=Your_Location_For_The_Singularity_directory/.` From 6480aef1db757943b09608212fcad01f16687a47 Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 11:41:33 +0100 Subject: [PATCH 143/181] typo --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 680feab..f4e66cb 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -82,7 +82,7 @@ For security reasons, there is no internet access on Bianca so you can't downloa You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. -After transferring the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : +Next transfer the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : `export NXF_SINGULARITY_CACHEDIR=Your_Location_For_The_Singularity_directory/.` From febf38dd8ecaba8d0000643f37d7db024d4ce969 Mon Sep 17 00:00:00 2001 From: Emelie Nilsso Date: Thu, 26 Nov 2020 13:38:47 +0100 Subject: [PATCH 144/181] Removed an old process so that only relevant processes are included --- conf/pipeline/ampliseq/uppmax.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conf/pipeline/ampliseq/uppmax.config b/conf/pipeline/ampliseq/uppmax.config index b6cfea2..2a8bc34 100644 --- a/conf/pipeline/ampliseq/uppmax.config +++ b/conf/pipeline/ampliseq/uppmax.config @@ -6,12 +6,6 @@ params { } process { - //"make_classifier" is the older version (just one step) of preparing the database for classification - withName: make_classifier { - clusterOptions = { "-A $params.project -C fat -p node -N 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } - } - - //"classifier_extract_seq" and "classifier_train" is the newer version, where database preparation is split in two to optimise resources withName: classifier_extract_seq { clusterOptions = { "-A $params.project -p core -n 1 -t 7-00:00:00 ${params.clusterOptions ?: ''}" } } From c2c726551811ffdcb69f0d6906e0d1f65f7ccda6 Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 13:47:47 +0100 Subject: [PATCH 145/181] fix linting errors --- docs/uppmax.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index f4e66cb..43c6282 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -74,14 +74,12 @@ It is not suitable for use with real data. To use it, submit with `-profile uppmax,devel`. - ## Running on Bianca For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on Bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your Bianca project. You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. - Next transfer the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : `export NXF_SINGULARITY_CACHEDIR=Your_Location_For_The_Singularity_directory/.` From ac108379cd02ee9ca9c611f516e23a38b3f68b44 Mon Sep 17 00:00:00 2001 From: arontommi Date: Thu, 26 Nov 2020 13:58:16 +0100 Subject: [PATCH 146/181] trailing spaces --- docs/uppmax.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 43c6282..d9ac5ce 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -76,9 +76,9 @@ To use it, submit with `-profile uppmax,devel`. ## Running on Bianca -For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on Bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your Bianca project. +For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on Bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your Bianca project. -You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. +You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. Next transfer the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : From 5590a9743ad2e5d185d9e074e98bcc2098d2a64d Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Thu, 26 Nov 2020 15:05:25 +0200 Subject: [PATCH 147/181] Update ebc.config queueSize to 64 --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index 4df0596..f159487 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -16,7 +16,7 @@ beforeScript = 'module load nextflow' } executor { - queueSize = 16 + queueSize = 64 } params { max_memory = 12.GB From c9ceaff3b3fecb72240fd54dbd14447a72b85135 Mon Sep 17 00:00:00 2001 From: marcel-keller <61977721+marcel-keller@users.noreply.github.com> Date: Wed, 9 Dec 2020 15:11:21 +0200 Subject: [PATCH 148/181] change of directory for conda environments change of directory due to recent changes in our infrastructure --- conf/ebc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ebc.config b/conf/ebc.config index f159487..8f007ed 100644 --- a/conf/ebc.config +++ b/conf/ebc.config @@ -8,7 +8,7 @@ cleanup = true conda { - cacheDir = '/ebc_data/nf-core/conda' + cacheDir = '/gpfs/space/GI/ebc_data/software/nf-core/conda' } process { executor = 'slurm' From a61992d144c35aeed9adf08b69239ec97f0034fc Mon Sep 17 00:00:00 2001 From: Combiz Khozoie Date: Fri, 18 Dec 2020 10:21:26 +0000 Subject: [PATCH 149/181] Added institutional configs for Imperial and Imperial MEDBIO. Added institutional pipeline configs for scflow for Imperial and Imperial MEDBIO --- conf/imperial.config | 39 +++++++++++++++++++++++ conf/imperial_mb.config | 47 ++++++++++++++++++++++++++++ conf/pipeline/scflow/imperial.config | 18 +++++++++++ docs/imperial.md | 16 ++++++++++ docs/imperial_mb.md | 16 ++++++++++ docs/pipeline/scflow/imperial.md | 21 +++++++++++++ nfcore_custom.config | 4 +++ pipeline/scflow.config | 14 +++++++++ 8 files changed, 175 insertions(+) create mode 100644 conf/imperial.config create mode 100644 conf/imperial_mb.config create mode 100644 conf/pipeline/scflow/imperial.config create mode 100644 docs/imperial.md create mode 100644 docs/imperial_mb.md create mode 100644 docs/pipeline/scflow/imperial.md create mode 100644 pipeline/scflow.config diff --git a/conf/imperial.config b/conf/imperial.config new file mode 100644 index 0000000..6308a69 --- /dev/null +++ b/conf/imperial.config @@ -0,0 +1,39 @@ +//Profile config names for nf-core/configs + +params { + // Config Params + config_profile_description = 'Imperial College London - HPC Profile -- provided by nf-core/configs.' + config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)' + config_profile_url = 'https://www.imperial.ac.uk/admin-services/ict/self-service/research-support/rcs/' + + // Resources + max_memory = 256.GB + max_cpus = 32 + max_time = 72.h +} + +workDir = "/rds/general/user/$USER/ephemeral/tmp" + +executor { + $pbspro { + queueSize = 50 + } + + $local { + cpus = 2 + queueSize = 1 + memory = '32 GB' + } +} + +singularity { + enabled = true + autoMounts = true + runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp" +} + +process { + + executor = 'pbspro' + +} \ No newline at end of file diff --git a/conf/imperial_mb.config b/conf/imperial_mb.config new file mode 100644 index 0000000..11b337a --- /dev/null +++ b/conf/imperial_mb.config @@ -0,0 +1,47 @@ +//Profile config names for nf-core/configs + +params { + // Config Params + config_profile_description = 'Imperial College London - MEDBIO QUEUE - HPC Profile -- provided by nf-core/configs.' + config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)' + config_profile_url = 'https://www.imperial.ac.uk/bioinformatics-data-science-group/resources/uk-med-bio/' + + // Resources + max_memory = 640.GB + max_cpus = 32 + max_time = 168.h +} + +workDir = "/rds/general/user/$USER/ephemeral/tmp" + +executor { + $pbspro { + queueSize = 50 + } + + $local { + cpus = 2 + queueSize = 1 + memory = '32 GB' + } +} + +singularity { + enabled = true + autoMounts = true + runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp" +} + +process { + + executor = 'pbspro' + queue = 'pqmedbio-tput' + + //queue = 'med-bio' //!! this is an alias and shouldn't be used + + withLabel:process_large { + queue = 'pqmedbio-large' + } + + +} \ No newline at end of file diff --git a/conf/pipeline/scflow/imperial.config b/conf/pipeline/scflow/imperial.config new file mode 100644 index 0000000..7f46466 --- /dev/null +++ b/conf/pipeline/scflow/imperial.config @@ -0,0 +1,18 @@ +// scflow/imperial specific profile config + +params { + // Config Params + config_profile_description = 'Imperial College London - HPC - nf-core/scFlow Profile -- provided by nf-core/configs.' + config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)' + + // Analysis Resource Params + ctd_folder = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd" + ensembl_mappings = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv" +} + +singularity { + enabled = true + autoMounts = true + cacheDir = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/.singularity-cache" + runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp" +} \ No newline at end of file diff --git a/docs/imperial.md b/docs/imperial.md new file mode 100644 index 0000000..43fc117 --- /dev/null +++ b/docs/imperial.md @@ -0,0 +1,16 @@ +# nf-core/configs: Imperial CX1 HPC Configuration + +All nf-core pipelines have been successfully configured for use on the CX1 cluster at Imperial College London HPC. + +To use, run the pipeline with `-profile imperial`. This will download and launch the [`imperial.config`](../conf/imperial.config) which has been pre-configured with a setup suitable for the CX1 cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow using the environment module system on the CX1 cluster. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module load Nextflow +``` + +>NB: You will need an account to use the HPC cluster CX1 in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. +>NB: To submit jobs to the Imperial College MEDBIO cluster, use `-profile imperial_mb` instead. diff --git a/docs/imperial_mb.md b/docs/imperial_mb.md new file mode 100644 index 0000000..d7f7f15 --- /dev/null +++ b/docs/imperial_mb.md @@ -0,0 +1,16 @@ +# nf-core/configs: Imperial MEDBIO HPC Configuration + +All nf-core pipelines have been successfully configured for use on the MEDBIO cluster at Imperial College London HPC. + +To use, run the pipeline with `-profile imperial_mb`. This will download and launch the [`imperial_mb.config`](../conf/imperial_mb.config) which has been pre-configured with a setup suitable for the MEDBIO cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow using the environment module system on the head node. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module load Nextflow +``` + +>NB: You will need an account to use the HPC cluster MEDBIO in order to run the pipeline. Access to the MEDBIO queue is exclusive. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. +>NB: To submit jobs to the standard CX1 cluster at Imperial College, use `-profile imperial` instead. diff --git a/docs/pipeline/scflow/imperial.md b/docs/pipeline/scflow/imperial.md new file mode 100644 index 0000000..be2cc59 --- /dev/null +++ b/docs/pipeline/scflow/imperial.md @@ -0,0 +1,21 @@ +# nf-core/configs: Imperial scflow Specific Configuration + +Extra specific configuration for the scflow pipeline + +## Usage + +To use, run the pipeline with `-profile imperial` or `-profile imperial_mb`. + +This will download and launch the scflow specific [`imperial.config`](../../../conf/pipeline/scflow/imperial.config) which has been pre-configured with a setup suitable for the Imperial HPC cluster. + +Example: `nextflow run nf-core/scflow -profile imperial` + +## scflow specific configurations for Imperial + +Specific configurations for Imperial have been made for scflow. + +* Singularity `enabled` and `autoMounts` set to `true` +* Singularity `cacheDir` path set to an RDS location +* Singularity `runOptions` path set to bind (`-B`) RDS paths with container paths. +* Params `ctd_folder` set to an RDS location. +* Parms `ensembl_mappings` set to an RDS location. diff --git a/nfcore_custom.config b/nfcore_custom.config index 2b3c419..e31c38e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,6 +24,8 @@ profiles { czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } + imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } + imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } google { includeConfig "${params.custom_config_base}/conf/google.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } @@ -54,6 +56,8 @@ params { cfc: ['.hpc.uni-tuebingen.de'], crick: ['.thecrick.org'], icr_davros: ['.davros.compute.estate'], + imperial: ['.hpc.ic.ac.uk'], + imperial_mb: ['.hpc.ic.ac.uk'], genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'], diff --git a/pipeline/scflow.config b/pipeline/scflow.config new file mode 100644 index 0000000..8c4697f --- /dev/null +++ b/pipeline/scflow.config @@ -0,0 +1,14 @@ +/* + * ------------------------------------------------- + * nfcore/scflow custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/scflow folder and imported + * under a profile name here. + */ + +profiles { + imperial { includeConfig "${params.custom_config_base}/conf/pipeline/scflow/imperial.config" } + imperial_mb { includeConfig "${params.custom_config_base}/conf/pipeline/scflow/imperial.config" } // intended +} \ No newline at end of file From 4bb52358f5698dd23b9a4854c95547df25c7a3a2 Mon Sep 17 00:00:00 2001 From: Combiz Khozoie Date: Fri, 18 Dec 2020 10:27:40 +0000 Subject: [PATCH 150/181] updated Github Actions YML --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dfb0782..e78bddd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 3824403b4e4b9a675919b46ea07a1e485a040940 Mon Sep 17 00:00:00 2001 From: Combiz Khozoie Date: Fri, 18 Dec 2020 10:31:36 +0000 Subject: [PATCH 151/181] removed workDir from Imperial configs --- conf/imperial.config | 2 -- conf/imperial_mb.config | 2 -- 2 files changed, 4 deletions(-) diff --git a/conf/imperial.config b/conf/imperial.config index 6308a69..0ca2498 100644 --- a/conf/imperial.config +++ b/conf/imperial.config @@ -12,8 +12,6 @@ params { max_time = 72.h } -workDir = "/rds/general/user/$USER/ephemeral/tmp" - executor { $pbspro { queueSize = 50 diff --git a/conf/imperial_mb.config b/conf/imperial_mb.config index 11b337a..eabeb83 100644 --- a/conf/imperial_mb.config +++ b/conf/imperial_mb.config @@ -12,8 +12,6 @@ params { max_time = 168.h } -workDir = "/rds/general/user/$USER/ephemeral/tmp" - executor { $pbspro { queueSize = 50 From 17cf4e3b35799592e48cfc79ccf32fd5c69abfdf Mon Sep 17 00:00:00 2001 From: Combiz Khozoie Date: Fri, 18 Dec 2020 15:34:56 +0000 Subject: [PATCH 152/181] fixed indents --- conf/imperial.config | 18 +++++++++--------- conf/imperial_mb.config | 19 +++++++++---------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/conf/imperial.config b/conf/imperial.config index 0ca2498..f40d92b 100644 --- a/conf/imperial.config +++ b/conf/imperial.config @@ -13,15 +13,15 @@ params { } executor { - $pbspro { - queueSize = 50 - } + $pbspro { + queueSize = 50 + } - $local { - cpus = 2 - queueSize = 1 - memory = '32 GB' - } + $local { + cpus = 2 + queueSize = 1 + memory = '32 GB' + } } singularity { @@ -34,4 +34,4 @@ process { executor = 'pbspro' -} \ No newline at end of file +} diff --git a/conf/imperial_mb.config b/conf/imperial_mb.config index eabeb83..a89ffcd 100644 --- a/conf/imperial_mb.config +++ b/conf/imperial_mb.config @@ -13,15 +13,15 @@ params { } executor { - $pbspro { - queueSize = 50 - } + $pbspro { + queueSize = 50 + } - $local { - cpus = 2 - queueSize = 1 - memory = '32 GB' - } + $local { + cpus = 2 + queueSize = 1 + memory = '32 GB' + } } singularity { @@ -41,5 +41,4 @@ process { queue = 'pqmedbio-large' } - -} \ No newline at end of file +} From a4c55b3ea93a552cf12978ae0033b28bbe385c16 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 6 Jan 2021 10:44:12 +0100 Subject: [PATCH 153/181] Fix time limit 0-9 typo --- conf/pipeline/eager/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index b358712..8accfa7 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -59,7 +59,7 @@ process { withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 2.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } } From 9b5ac5c89d1761e877956c55e4a0a74af512a6a6 Mon Sep 17 00:00:00 2001 From: rbpisupati Date: Fri, 15 Jan 2021 12:55:46 +0100 Subject: [PATCH 154/181] added anaconda module for cbe --- conf/cbe.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/cbe.config b/conf/cbe.config index 0e9b43e..0a5763f 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -9,6 +9,7 @@ process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } + module = 'anaconda3/2019.10' } singularity { From 1c25cb033b3990c7c71b7fd1bd7e352155e79d3f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 19 Jan 2021 13:53:01 +0100 Subject: [PATCH 155/181] Add big_data profile --- conf/pipeline/eager/shh.config | 65 +++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index b358712..bdc3470 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -59,12 +59,75 @@ process { withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1449.h : task.attempt == 2 ? 48.h : 2.h } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } } profiles { + + big_data { + + executor { + queueSize = 6 + } + + process { + + maxRetries = 2 + + withName: malt { + maxRetries = 1 + memory = { task.attempt > 1 ? 1900.GB : 725.GB } + cpus = { task.attempt > 1 ? 112 : 64 } + time = 1440.h + queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } + } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 2.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 512.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + } + } + microbiome_screening { process { // Assuming NCBI NT-sized DB From c11c26f15ffa59d576acaa43a931a84b06120903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Wed, 20 Jan 2021 23:45:26 +0100 Subject: [PATCH 156/181] Create ceh.config --- conf/ceh.config | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 conf/ceh.config diff --git a/conf/ceh.config b/conf/ceh.config new file mode 100644 index 0000000..1c54682 --- /dev/null +++ b/conf/ceh.config @@ -0,0 +1,31 @@ +//Profile config names for nf-core/configs + params { + config_profile_description = 'Centre for Evolutionary Hologenomics - CEH profile provided by nf-core/configs.' + config_profile_contact = 'Aashild Vaagene (@ashildv)' + } + +profiles { +ceh { + singularity { + enabled = true + autoMounts = true + cacheDir = '/shared/volume/hologenomics/data/cache/nf-eager/singularity' + } + process { + executor = 'slurm' + queue = { task.time < 24.h ? 'hologenomics-short' : task.time < 168.h ? 'hologenomics' : 'hologenomics-long' } + } + + cleanup = true + + executor { + queueSize = 8 + } + params { + max_memory = 250.GB + max_cpus = 35 + max_time = 720.h + } + } + } + From 8a8f95e364e8d0585053c927c1d596a9ca11862b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Wed, 20 Jan 2021 23:47:58 +0100 Subject: [PATCH 157/181] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index e31c38e..2519ee5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,6 +18,7 @@ profiles { cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } + ceh { includeConfig "${params.custom_config_base}/conf/ceh.config"} cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } From 65831b73ef4c7e1257350810777f98ef46000d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:17:08 +0100 Subject: [PATCH 158/181] Create ceh.md --- docs/ceh.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/ceh.md diff --git a/docs/ceh.md b/docs/ceh.md new file mode 100644 index 0000000..b9c8b4f --- /dev/null +++ b/docs/ceh.md @@ -0,0 +1,21 @@ +# nf-core/configs: Centre for Evolutionary Hologenomics / EvoGenomics (hologenomics partition on HPC) Configuration + +The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. + +Before running the pipeline you will need to load Java, Miniconda and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: + +```bash +## Load Java, Miniconda and Nextflow environment modules +module purge +module load lib +module load java/v1.8.0_202-jdk miniconda nextflow/v20.07.1.5412 +``` + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. +The config contains a `cleanup` command that removes the `work/` directory automatically once the pipeline has completeed successfully. If the run does not complete successfully then the `work/` dir should be removed manually to save storage space. + +This configuration will automatically choose the correct SLURM queue (short,medium,long) depending on the time and memory required by each process. + +>NB: You will need an account to use the HPC cluster to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be submitted from one of the login nodes. From a6f316b58b6c83cee74a6f62572c2a9ce590805a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:21:13 +0100 Subject: [PATCH 159/181] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a967049..42d2b91 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,7 @@ Currently documentation is available for the following systems: * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) +* [CEH] (docs/ceh.md) * [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) From 22e6a60db10edf5b19c0036a5d6ba2fa1e14ecf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:22:13 +0100 Subject: [PATCH 160/181] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42d2b91..1c1bcc4 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Currently documentation is available for the following systems: * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) -* [CEH] (docs/ceh.md) +* [CEH](docs/ceh.md) * [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) From 91594dffeaf5f9eceeb22ccf1e011ab2767627a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:29:43 +0100 Subject: [PATCH 161/181] Update ceh.config --- conf/ceh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ceh.config b/conf/ceh.config index 1c54682..ff81cfb 100644 --- a/conf/ceh.config +++ b/conf/ceh.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Centre for Evolutionary Hologenomics - CEH profile provided by nf-core/configs.' + config_profile_description = 'Center for Evolutionary Hologenomics / Section for Evolutionary Genomics @ GLOBE - CEH profile provided by nf-core/configs.' config_profile_contact = 'Aashild Vaagene (@ashildv)' } From d108666950703a521518535c38eddad32de372f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:30:54 +0100 Subject: [PATCH 162/181] Update ceh.md --- docs/ceh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ceh.md b/docs/ceh.md index b9c8b4f..fff2921 100644 --- a/docs/ceh.md +++ b/docs/ceh.md @@ -1,4 +1,4 @@ -# nf-core/configs: Centre for Evolutionary Hologenomics / EvoGenomics (hologenomics partition on HPC) Configuration +# nf-core/configs: Center for Evolutionary Hologenomics & Section for Evolutionary Genomics (hologenomics partition on HPC) Configuration The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. From f8973f153a59833ff727c1caac7ed7f5667e52db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 00:31:49 +0100 Subject: [PATCH 163/181] Update ceh.config --- conf/ceh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ceh.config b/conf/ceh.config index ff81cfb..ee15f32 100644 --- a/conf/ceh.config +++ b/conf/ceh.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Center for Evolutionary Hologenomics / Section for Evolutionary Genomics @ GLOBE - CEH profile provided by nf-core/configs.' + config_profile_description = 'Center for Evolutionary Hologenomics & Section for Evolutionary Genomics @ GLOBE - CEH profile provided by nf-core/configs.' config_profile_contact = 'Aashild Vaagene (@ashildv)' } From 964cae4ac79cab4a1e209f826d9897a37d8d0d48 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 21 Jan 2021 06:22:37 +0100 Subject: [PATCH 164/181] Add CEH to CI --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e78bddd..f9ffee0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ceh', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 2fd475c3398305e3377cc2e8af18cc5cb2c68180 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 21 Jan 2021 06:30:43 +0100 Subject: [PATCH 165/181] Fix some linting errors --- docs/ceh.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ceh.md b/docs/ceh.md index fff2921..11fe720 100644 --- a/docs/ceh.md +++ b/docs/ceh.md @@ -1,6 +1,8 @@ # nf-core/configs: Center for Evolutionary Hologenomics & Section for Evolutionary Genomics (hologenomics partition on HPC) Configuration -The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. +> **NB:** You will need an account to use the HPC cluster to run the pipeline. If in doubt contact IT. + +The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. Before running the pipeline you will need to load Java, Miniconda and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: @@ -16,6 +18,4 @@ The config contains a `cleanup` command that removes the `work/` directory autom This configuration will automatically choose the correct SLURM queue (short,medium,long) depending on the time and memory required by each process. ->NB: You will need an account to use the HPC cluster to run the pipeline. If in doubt contact IT. - ->NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be submitted from one of the login nodes. +> **NB:** Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be submitted from one of the login nodes. From 8fa720b9cc4afe6c38eb8bb44e00ad0a0e5074ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:15:06 +0100 Subject: [PATCH 166/181] Update conf/ceh.config Co-authored-by: James A. Fellows Yates --- conf/ceh.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/ceh.config b/conf/ceh.config index ee15f32..77389a7 100644 --- a/conf/ceh.config +++ b/conf/ceh.config @@ -2,6 +2,7 @@ params { config_profile_description = 'Center for Evolutionary Hologenomics & Section for Evolutionary Genomics @ GLOBE - CEH profile provided by nf-core/configs.' config_profile_contact = 'Aashild Vaagene (@ashildv)' + config_profile_url = 'https://ceh.ku.dk/ } profiles { From ef80452577b18aa40ed2e1e63eab62e0abe3c3c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:16:46 +0100 Subject: [PATCH 167/181] Update docs/ceh.md Co-authored-by: James A. Fellows Yates --- docs/ceh.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ceh.md b/docs/ceh.md index 11fe720..1381613 100644 --- a/docs/ceh.md +++ b/docs/ceh.md @@ -4,7 +4,7 @@ The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. -Before running the pipeline you will need to load Java, Miniconda and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: +Before running the pipeline you will need to load Java, and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: ```bash ## Load Java, Miniconda and Nextflow environment modules From 594985f1eeac1bbf0c9cd535fc6e60ce9faa466d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:43:34 +0100 Subject: [PATCH 168/181] Update and rename ceh.config to seg_globe.config --- conf/{ceh.config => seg_globe.config} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename conf/{ceh.config => seg_globe.config} (75%) diff --git a/conf/ceh.config b/conf/seg_globe.config similarity index 75% rename from conf/ceh.config rename to conf/seg_globe.config index 77389a7..390d0ef 100644 --- a/conf/ceh.config +++ b/conf/seg_globe.config @@ -1,8 +1,8 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Center for Evolutionary Hologenomics & Section for Evolutionary Genomics @ GLOBE - CEH profile provided by nf-core/configs.' + config_profile_description = 'Section for Evolutionary Genomics @ GLOBE, University of Copenhagen - seg_globe profile provided by nf-core/configs.' config_profile_contact = 'Aashild Vaagene (@ashildv)' - config_profile_url = 'https://ceh.ku.dk/ + config_profile_url = 'https://globe.ku.dk/research/evogenomics/' } profiles { From bcd137225e293a4cdc93a822f81c53f67e9ea3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:46:46 +0100 Subject: [PATCH 169/181] Update and rename ceh.md to seg_globe.md --- docs/{ceh.md => seg_globe.md} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename docs/{ceh.md => seg_globe.md} (74%) diff --git a/docs/ceh.md b/docs/seg_globe.md similarity index 74% rename from docs/ceh.md rename to docs/seg_globe.md index 1381613..1fa832f 100644 --- a/docs/ceh.md +++ b/docs/seg_globe.md @@ -1,16 +1,16 @@ -# nf-core/configs: Center for Evolutionary Hologenomics & Section for Evolutionary Genomics (hologenomics partition on HPC) Configuration +# nf-core/configs: Section for Evolutionary Genomics at GLOBE, Univeristy of Copenhagen (hologenomics partition on HPC) Configuration > **NB:** You will need an account to use the HPC cluster to run the pipeline. If in doubt contact IT. The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. -Before running the pipeline you will need to load Java, and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: +Before running the pipeline you will need to load Java and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: ```bash -## Load Java, Miniconda and Nextflow environment modules +## Load Java and Nextflow environment modules module purge module load lib -module load java/v1.8.0_202-jdk miniconda nextflow/v20.07.1.5412 +module load java/v1.8.0_202-jdk nextflow/v20.07.1.5412 ``` All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. From 8037b73ddd6a5a1fae3803c65915c4c27f1180ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:52:58 +0100 Subject: [PATCH 170/181] Update nfcore_custom.config --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 2519ee5..7310de5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,7 +18,6 @@ profiles { cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } - ceh { includeConfig "${params.custom_config_base}/conf/ceh.config"} cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } @@ -39,6 +38,7 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } + seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } From 765a2e0b7ed6460df9f980cfee350f5b7a7884f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=85shild=20J=2E=20V=C3=A5gene?= <60298098+ashildv@users.noreply.github.com> Date: Thu, 21 Jan 2021 11:55:00 +0100 Subject: [PATCH 171/181] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c1bcc4..a590599 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,6 @@ Currently documentation is available for the following systems: * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) -* [CEH](docs/ceh.md) * [CFC](docs/cfc.md) * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) @@ -120,6 +119,7 @@ Currently documentation is available for the following systems: * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) +* [SEG_GLOBE](docs/seg_globe.md) * [SHH](docs/shh.md) * [UCT_HPC](docs/uct_hpc.md) * [UPPMAX](docs/uppmax.md) From 7cffcf809bdd0070d317085923499bacc96c9068 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 21 Jan 2021 12:50:14 +0100 Subject: [PATCH 172/181] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9ffee0..ea2117d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'ceh', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From c4604450d864392741e37867f622298bf2fc4626 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 21 Jan 2021 13:04:12 +0100 Subject: [PATCH 173/181] Update seg_globe.md --- docs/seg_globe.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/seg_globe.md b/docs/seg_globe.md index 1fa832f..da03737 100644 --- a/docs/seg_globe.md +++ b/docs/seg_globe.md @@ -4,13 +4,13 @@ The profile is configured to run with Singularity version 3.6.3-1.el7 which is part of the OS installtion and does not need to be loaded as a module. -Before running the pipeline you will need to load Java and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: +Before running the pipeline you will need to load Java, miniconda and Nextflow. You can do this by including the commands below in your SLURM/sbatch script: ```bash ## Load Java and Nextflow environment modules module purge module load lib -module load java/v1.8.0_202-jdk nextflow/v20.07.1.5412 +module load java/v1.8.0_202-jdk miniconda nextflow/v20.07.1.5412 ``` All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. From 26d164ca74ccc25ba7ee1e6e8fb9843ee7563f65 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 21 Jan 2021 13:15:08 +0100 Subject: [PATCH 174/181] Remove ceh profile for now --- conf/seg_globe.config | 53 ++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/conf/seg_globe.config b/conf/seg_globe.config index 390d0ef..41a3d6e 100644 --- a/conf/seg_globe.config +++ b/conf/seg_globe.config @@ -1,32 +1,27 @@ //Profile config names for nf-core/configs - params { - config_profile_description = 'Section for Evolutionary Genomics @ GLOBE, University of Copenhagen - seg_globe profile provided by nf-core/configs.' - config_profile_contact = 'Aashild Vaagene (@ashildv)' - config_profile_url = 'https://globe.ku.dk/research/evogenomics/' - } +params { + config_profile_description = 'Section for Evolutionary Genomics @ GLOBE, University of Copenhagen - seg_globe profile provided by nf-core/configs.' + config_profile_contact = 'Aashild Vaagene (@ashildv)' + config_profile_url = 'https://globe.ku.dk/research/evogenomics/' + max_memory = 250.GB + max_cpus = 35 + max_time = 720.h +} -profiles { -ceh { - singularity { - enabled = true - autoMounts = true - cacheDir = '/shared/volume/hologenomics/data/cache/nf-eager/singularity' - } - process { - executor = 'slurm' - queue = { task.time < 24.h ? 'hologenomics-short' : task.time < 168.h ? 'hologenomics' : 'hologenomics-long' } - } +singularity { + enabled = true + autoMounts = true + cacheDir = '/shared/volume/hologenomics/data/cache/nf-eager/singularity' +} + +process { + executor = 'slurm' + queue = { task.time < 24.h ? 'hologenomics-short' : task.time < 168.h ? 'hologenomics' : 'hologenomics-long' } +} - cleanup = true - - executor { - queueSize = 8 - } - params { - max_memory = 250.GB - max_cpus = 35 - max_time = 720.h - } - } - } - +cleanup = true + +executor { + queueSize = 8 +} + From 713a032a36d22b96e1a2cae96a3c78fa6b40f3fb Mon Sep 17 00:00:00 2001 From: "Thiseas C. Lamnidis" Date: Tue, 26 Jan 2021 15:08:21 +0100 Subject: [PATCH 175/181] Add debug profile which deactivates cleanup of workdir after successful run --- conf/shh.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index dc51e87..ab3dcca 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -10,6 +10,7 @@ params { igenomes_base = "/projects1/public_data/igenomes/" } +// Preform work directory cleanup after a successful run cleanup = true singularity { @@ -46,4 +47,8 @@ profiles { queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } } + // Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option. + debug { + cleanup = false + } } From 1608b36dcf0624302b065f60bb75b9b2f23ce487 Mon Sep 17 00:00:00 2001 From: Charles Plessy Date: Tue, 2 Feb 2021 17:40:10 +0900 Subject: [PATCH 176/181] Institutional profile for Okinawa Institute of Science and Technology --- .github/workflows/main.yml | 2 +- README.md | 1 + conf/oist.config | 22 ++++++++++++++++++++++ docs/oist.md | 33 +++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 5 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 conf/oist.config create mode 100644 docs/oist.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea2117d..3e5c930 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index a590599..45721e8 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ Currently documentation is available for the following systems: * [KRAKEN](docs/kraken.md) * [MPCDF](docs/mpcdf.md) * [MUNIN](docs/munin.md) +* [OIST](docs/oist.md) * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) diff --git a/conf/oist.config b/conf/oist.config new file mode 100644 index 0000000..8815ed4 --- /dev/null +++ b/conf/oist.config @@ -0,0 +1,22 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Okinawa Institute of Science and Technology Graduate University (OIST) HPC cluster profile provided by nf-core/configs.' + config_profile_contact = 'OISTs Bioinformatics User Group ' + config_profile_url = 'https://github.com/nf-core/configs/blob/master/docs/oist.md' +} + +singularity { + enabled = true +} + +process { + executor = 'slurm' + queue = 'compute' + clusterOptions = '-C zen2' +} + +params { + max_memory = 500.GB + max_cpus = 128 + max_time = 90.h +} diff --git a/docs/oist.md b/docs/oist.md new file mode 100644 index 0000000..8c4a68c --- /dev/null +++ b/docs/oist.md @@ -0,0 +1,33 @@ +# nf-core/configs: OIST Configuration + +The nf-core pipelines [rnaseq](https://nf-co.re/rnaseq) and +[eager](https://nf-co.re/eager) have been successfully tested on the _Deigo_ +cluster at the Okinawa Institute of Science and Technology Graduate University +([OIST](https://www.oist.jp)). We have no reason to expect that other +pipelines would not work. + +To use, run the pipeline with `-profile oist`. This will download and launch +the [`oist.config`](../conf/oist.config) which has been pre-configured with a +setup suitable for _Deigo_. Using this profile, a docker image containing all +of the required software will be downloaded, and converted to a Singularity +image before execution of the pipeline. + +## Below are non-mandatory information e.g. on modules to load etc + +Before running the pipeline you will need to load Nextflow and Singularity +using the environment module system on _Deigo_. You can do this by issuing the +commands below: + +```bash +## Load the latest Nextflow and Singularity environment modules +ml purge +ml bioinfo-ugrp-modules +ml Other/Nextflow +``` + +>NB: You will need an account to use the _Deigo_ cluster in order to run the +>pipeline. If in doubt contact IT. +> +>NB: Nextflow will submit the jobs via the SLURM scheduler to the HPC cluster +>and as such the commands above will have to be executed on one of the login +>nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 7310de5..429e3c1 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -35,6 +35,7 @@ profiles { kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } mpcdf { includeConfig "${params.custom_config_base}/conf/mpcdf.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } + oist { includeConfig "${params.custom_config_base}/conf/oist.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } From eb725f9ea35bae0e534847618275fc8f64423176 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 3 Feb 2021 20:13:09 +0100 Subject: [PATCH 177/181] Update shh.config --- conf/pipeline/eager/shh.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index bdc3470..2c7ec82 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -83,6 +83,12 @@ profiles { time = 1440.h queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } + + withName: hostremoval_input_fastq { + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } From e7ac0bae138a95de436c550b172b47d8b0d0e0ac Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 4 Feb 2021 08:00:57 +0100 Subject: [PATCH 178/181] Update shh.config --- conf/pipeline/eager/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 2c7ec82..787e5c5 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -68,6 +68,10 @@ profiles { big_data { + params { + config_profile_description = 'nf-core/eager 'big-data' SHH profile provided by nf-core/configs' + } + executor { queueSize = 6 } From 82fe8a04be8287f43f690bfdc1dafb1c7602345a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 4 Feb 2021 08:02:33 +0100 Subject: [PATCH 179/181] Update shh.config --- conf/pipeline/eager/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 787e5c5..0ff23a1 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -69,7 +69,7 @@ profiles { big_data { params { - config_profile_description = 'nf-core/eager 'big-data' SHH profile provided by nf-core/configs' + config_profile_description = 'nf-core/eager big-data SHH profile provided by nf-core/configs' } executor { From c515a401c705896cc2158ddb15846ccf279116a9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 4 Feb 2021 08:06:35 +0100 Subject: [PATCH 180/181] Update shh.config --- conf/pipeline/eager/shh.config | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 0ff23a1..ab0d0fb 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -69,7 +69,9 @@ profiles { big_data { params { - config_profile_description = 'nf-core/eager big-data SHH profile provided by nf-core/configs' + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager big-data SHH profile provided by nf-core/configs' } executor { @@ -80,7 +82,7 @@ profiles { maxRetries = 2 - withName: malt { + withName: 'malt' { maxRetries = 1 memory = { task.attempt > 1 ? 1900.GB : 725.GB } cpus = { task.attempt > 1 ? 112 : 64 } @@ -88,7 +90,7 @@ profiles { queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } - withName: hostremoval_input_fastq { + withName: 'hostremoval_input_fastq' { cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } From e32f1e46ff77577538250b85fedde397cb23003e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 4 Feb 2021 08:13:16 +0100 Subject: [PATCH 181/181] Update shh.config --- conf/pipeline/eager/shh.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index ab0d0fb..44d1c3c 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -82,7 +82,7 @@ profiles { maxRetries = 2 - withName: 'malt' { + withName:malt { maxRetries = 1 memory = { task.attempt > 1 ? 1900.GB : 725.GB } cpus = { task.attempt > 1 ? 112 : 64 } @@ -90,10 +90,11 @@ profiles { queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } - withName: 'hostremoval_input_fastq' { + withName:hostremoval_input_fastq { cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + time = 1440.h + queue = 'long' } withLabel:'sc_tiny'{