From 94f8cbcb847679ef7fb6aa295e54b5f676905962 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 3 Mar 2022 12:20:02 +0100 Subject: [PATCH 1/4] Add rnaseq pipeline specific config for MPCDF --- conf/pipeline/rnaseq/mpcdf.config | 51 +++++++++++++++++++++++++++++++ docs/mpcdf.md | 15 +++++++-- pipeline/eager.config | 2 +- pipeline/rnaseq.config | 1 + 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 conf/pipeline/rnaseq/mpcdf.config diff --git a/conf/pipeline/rnaseq/mpcdf.config b/conf/pipeline/rnaseq/mpcdf.config new file mode 100644 index 0000000..6954946 --- /dev/null +++ b/conf/pipeline/rnaseq/mpcdf.config @@ -0,0 +1,51 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/rnaseq MPCDF profile provided by nf-core/configs' +} + +profiles { + cobra { + process { + cpus = { check_max( 1 * task.attempt, 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + + errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + maxRetries = 1 + maxErrors = '-1' + + // Process-specific resource requirements + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + withLabel:process_medium { + cpus = 40 + memory = { check_max( 36.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + withLabel:process_high { + cpus = 40 + memory = { check_max( 72.GB * task.attempt, 'memory' ) } + time = { check_max( 10.h * task.attempt, 'time' ) } + } + withLabel:process_long { + time = 24.h + } + withLabel:process_high_memory { + cpus = 40 + memory = { check_max( 200.GB * task.attempt, 'memory' ) } + } + withLabel:error_ignore { + errorStrategy = 'ignore' + } + withName:CUSTOM_DUMPSOFTWAREVERSIONS { + cache = false + } + } + } +} \ No newline at end of file diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 2f9f55c..3c76af4 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -12,7 +12,8 @@ All profiles use `singularity` as the corresponding containerEngine. To prevent >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). -## cobra +## Global Profiles +### cobra To use: `-profile cobra,mpcdf` @@ -23,7 +24,7 @@ Sets the following parameters: - Max. CPUs: 80 - Max. walltime: 24.h -## raven +### raven To use: `-profile raven,mpcdf` @@ -33,3 +34,13 @@ Sets the following parameters: - Max. memory: 368.GB - Max. CPUs: 192 - Max. walltime: 24.h + +## Pipeline Profiles + +### nf-core/eager + +We offer a pipeline specific profile optimised for [nf-core/eager](https://nf-co.re/eager) + +### nf-core/rnaseq + +We offer a pipeline specific profile optimised for [nf-core/rnaseq](https://nf-co.re/rnaseq) diff --git a/pipeline/eager.config b/pipeline/eager.config index 10dd990..82fd071 100644 --- a/pipeline/eager.config +++ b/pipeline/eager.config @@ -9,7 +9,7 @@ */ profiles { - mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } eva { includeConfig "${params.custom_config_base}/conf/pipeline/eager/eva.config" } maestro { includeConfig "${params.custom_config_base}/conf/pipeline/eager/maestro.config" } + mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } } diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config index 17a4ca0..0486d86 100644 --- a/pipeline/rnaseq.config +++ b/pipeline/rnaseq.config @@ -10,5 +10,6 @@ profiles { eddie { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/eddie.config" } + mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/mpcdf.config" } utd_sysbio { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/utd_sysbio.config" } } From 17312604cf29771a1cc23ce6a9165bcf1e6996b8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Mar 2022 12:25:06 +0100 Subject: [PATCH 2/4] Update mpcdf.config --- conf/pipeline/rnaseq/mpcdf.config | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/rnaseq/mpcdf.config b/conf/pipeline/rnaseq/mpcdf.config index 6954946..f26de1f 100644 --- a/conf/pipeline/rnaseq/mpcdf.config +++ b/conf/pipeline/rnaseq/mpcdf.config @@ -1,13 +1,12 @@ // Profile config names for nf-core/configs -params { - // Specific nf-core/configs params - config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/rnaseq MPCDF profile provided by nf-core/configs' -} - profiles { cobra { + params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/rnaseq MPCDF cobra profile provided by nf-core/configs' + } process { cpus = { check_max( 1 * task.attempt, 'cpus' ) } memory = { check_max( 6.GB * task.attempt, 'memory' ) } @@ -48,4 +47,4 @@ profiles { } } } -} \ No newline at end of file +} From e649bfbb2e22ea327e9485d93d3485dea9cdbbf0 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 3 Mar 2022 12:26:14 +0100 Subject: [PATCH 3/4] Linting --- docs/mpcdf.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 3c76af4..5aa7ad2 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -13,6 +13,7 @@ All profiles use `singularity` as the corresponding containerEngine. To prevent >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). ## Global Profiles + ### cobra To use: `-profile cobra,mpcdf` From 7112e74b73303cdbdd8decdeb5ffc2c3572918fe Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 3 Mar 2022 13:43:40 +0100 Subject: [PATCH 4/4] Tweak settings based on latest batch specs --- conf/mpcdf.config | 4 ++-- docs/mpcdf.md | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 281e7bf..88617d6 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -32,7 +32,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile (unofficially) provided by nf-core/configs.' max_memory = 725.GB - max_cpus = 80 + max_cpus = 40 max_time = 24.h } } @@ -61,7 +61,7 @@ profiles { params { config_profile_description = 'MPCDF raven profile (unofficially) provided by nf-core/configs.' - max_memory = 368.GB + max_memory = 2000000.MB max_cpus = 72 max_time = 24.h } diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 5aa7ad2..f1b1a4b 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -16,24 +16,24 @@ All profiles use `singularity` as the corresponding containerEngine. To prevent ### cobra -To use: `-profile cobra,mpcdf` +To use: `-profile mpcdf,cobra` Sets the following parameters: - Maximum parallel running jobs: 8 -- Max. memory: 750.GB -- Max. CPUs: 80 +- Max. memory: 725.GB +- Max. CPUs: 40 - Max. walltime: 24.h ### raven -To use: `-profile raven,mpcdf` +To use: `-profile mpcdf,raven` Sets the following parameters: - Maximum parallel running jobs: 8 -- Max. memory: 368.GB -- Max. CPUs: 192 +- Max. memory: 200000.MB (2.TB) +- Max. CPUs: 72 - Max. walltime: 24.h ## Pipeline Profiles