1
0
Fork 0
mirror of https://github.com/MillironX/nf-configs.git synced 2024-11-21 16:16:04 +00:00

Merge pull request #339 from jfy133/rnaseq_mpcdf

Add RNASEQ specific config for MPCDF
This commit is contained in:
Alexander Peltzer 2022-03-07 13:59:08 +01:00 committed by GitHub
commit d0d86bfb2a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 185 additions and 63 deletions

View file

@ -32,7 +32,7 @@ profiles {
params {
config_profile_description = 'MPCDF cobra profile (unofficially) provided by nf-core/configs.'
max_memory = 725.GB
max_cpus = 80
max_cpus = 40
max_time = 24.h
}
}
@ -47,7 +47,7 @@ profiles {
}
executor {
queueSize = 8
queueSize = 30
pollInterval = '1 min'
queueStatInterval = '5 min'
}
@ -61,7 +61,7 @@ profiles {
params {
config_profile_description = 'MPCDF raven profile (unofficially) provided by nf-core/configs.'
max_memory = 368.GB
memory = 2000000.MB
max_cpus = 72
max_time = 24.h
}

View file

@ -1,64 +1,121 @@
// Profile config names for nf-core/configs
params {
// Specific nf-core/configs params
config_profile_contact = 'James Fellows Yates (@jfy133)'
config_profile_description = 'nf-core/eager MPCDF profile provided by nf-core/configs'
}
profile {
cobra {
// Specific nf-core/eager process configuration
process {
cobra {
params {
// Specific nf-core/configs params
config_profile_contact = 'James Fellows Yates (@jfy133)'
config_profile_description = 'nf-core/eager MPCDF cobra profile provided by nf-core/configs'
}
process {
withName: malt {
maxRetries = 1
memory = 725.GB
cpus = 40
time = 24.h
}
withLabel:'sc_tiny'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 1.GB * task.attempt, 'memory' ) }
time = 24.h
}
withName: malt {
maxRetries = 1
memory = 725.GB
cpus = 40
time = 24.h
}
withLabel:'sc_tiny'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 1.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_small'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_small'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_medium'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_medium'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_small'{
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_small'{
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_medium' {
cpus = { check_max( 4 * task.attempt, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_medium' {
cpus = { check_max( 4 * task.attempt, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_large'{
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_large'{
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_huge'{
cpus = { check_max( 32 * task.attempt, 'cpus' ) }
memory = { check_max( 256.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_huge'{
cpus = { check_max( 32 * task.attempt, 'cpus' ) }
memory = { check_max( 256.GB * task.attempt, 'memory' ) }
time = 24.h
}
}
}
raven {
// Specific nf-core/eager process configuration
params {
// Specific nf-core/configs params
config_profile_contact = 'James Fellows Yates (@jfy133)'
config_profile_description = 'nf-core/eager MPCDF raven profile provided by nf-core/configs'
}
process {
withName: malt {
maxRetries = 1
memory = 2000000.MB
cpus = 72
time = 24.h
}
withLabel:'sc_tiny'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 1.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_small'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'sc_medium'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_small'{
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_medium' {
cpus = { check_max( 4 * task.attempt, 'cpus' ) }
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_large'{
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
time = 24.h
}
withLabel:'mc_huge'{
cpus = { check_max( 72, 'cpus' ) }
memory = { check_max( 240.GB * task.attempt, 'memory' ) }
time = 24.h
}
}
}
}

View file

@ -0,0 +1,50 @@
// Profile config names for nf-core/configs
profiles {
cobra {
params {
// Specific nf-core/configs params
config_profile_contact = 'James Fellows Yates (@jfy133)'
config_profile_description = 'nf-core/rnaseq MPCDF cobra profile provided by nf-core/configs'
}
process {
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
time = { check_max( 4.h * task.attempt, 'time' ) }
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
maxRetries = 1
maxErrors = '-1'
// Process-specific resource requirements
withLabel:process_low {
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
time = { check_max( 4.h * task.attempt, 'time' ) }
}
withLabel:process_medium {
cpus = 40
memory = { check_max( 36.GB * task.attempt, 'memory' ) }
time = { check_max( 6.h * task.attempt, 'time' ) }
}
withLabel:process_high {
cpus = 40
memory = { check_max( 72.GB * task.attempt, 'memory' ) }
time = { check_max( 10.h * task.attempt, 'time' ) }
}
withLabel:process_long {
time = 24.h
}
withLabel:process_high_memory {
cpus = 40
memory = { check_max( 200.GB * task.attempt, 'memory' ) }
}
withLabel:error_ignore {
errorStrategy = 'ignore'
}
withName:CUSTOM_DUMPSOFTWAREVERSIONS {
cache = false
}
}
}
}

View file

@ -12,24 +12,38 @@ All profiles use `singularity` as the corresponding containerEngine. To prevent
>NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing).
## cobra
## Global Profiles
To use: `-profile cobra,mpcdf`
### cobra
To use: `-profile mpcdf,cobra`
Sets the following parameters:
- Maximum parallel running jobs: 8
- Max. memory: 750.GB
- Max. CPUs: 80
- Max. memory: 725.GB
- Max. CPUs: 40
- Max. walltime: 24.h
## raven
### raven
To use: `-profile raven,mpcdf`
To use: `-profile mpcdf,raven`
Sets the following parameters:
- Maximum parallel running jobs: 8
- Max. memory: 368.GB
- Max. CPUs: 192
- Max. memory: 2000000.MB (2.TB)
- Max. CPUs: 72
- Max. walltime: 24.h
## Pipeline Profiles
### nf-core/eager
We offer a pipeline specific profile optimised for [nf-core/eager](https://nf-co.re/eager)
- Maximum parallel running jobs: 30
### nf-core/rnaseq
We offer a pipeline specific profile optimised for [nf-core/rnaseq](https://nf-co.re/rnaseq)

View file

@ -9,7 +9,7 @@
*/
profiles {
mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" }
eva { includeConfig "${params.custom_config_base}/conf/pipeline/eager/eva.config" }
maestro { includeConfig "${params.custom_config_base}/conf/pipeline/eager/maestro.config" }
mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" }
}

View file

@ -10,5 +10,6 @@
profiles {
eddie { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/eddie.config" }
mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/mpcdf.config" }
utd_sysbio { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/utd_sysbio.config" }
}