From 42a1a159700c0ca5b551ccd97a9126ad0f767f2e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:11:56 +0200 Subject: [PATCH 01/23] Create mpcdf.config --- conf/mpcdf.config | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 conf/mpcdf.config diff --git a/conf/mpcdf.config b/conf/mpcdf.config new file mode 100644 index 0000000..9a109b0 --- /dev/null +++ b/conf/mpcdf.config @@ -0,0 +1,30 @@ +params { + config_profile_description = 'MPCDF HPC profiles provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://www.mpcdf.mpg.de/services/computing' +} + + +profiles { + cobra { + // Does not have singularity! Conda module must be used, but it is + // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} + // To create common cache dir (only until /u/ periodically cleared!) + + process { + beforeScript = 'module load anaconda/3/2020.02' + executor = 'slurm' + } + + executor { + queueSize =8 + } + + params { + config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' + max_memory = 750.GB + max_cpus = 80 + max_time = 24.h + } + } +} From 0501ae4c1a1f7c7d3421d3bf7075de75d9f3bfce Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:21:49 +0200 Subject: [PATCH 02/23] Started docs for mpcdf --- docs/mpcdf.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/mpcdf.md diff --git a/docs/mpcdf.md b/docs/mpcdf.md new file mode 100644 index 0000000..11d4474 --- /dev/null +++ b/docs/mpcdf.md @@ -0,0 +1,22 @@ +# nf-core/configs: MPCDF Configuration + +All nf-core pipelines have been successfully configured for use on the HPCs at [Max Planck Computing and Data Facility](https://www.mpcdf.mpg.de/). + +> :warning: these profiles are not officially supported by the MPCDF. + +To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. + +Currently the following clusters are supported: cobra, draco* (* coming soon) + +>NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). + +# cobra + +Cobra does not currently support singularity, therefore the anaconda/module should be loaded for use. + +Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). + +To use: `-profile cobra,mpcdf` + +# draco + From a1c5dc5fde9e3877166488dcdb9a0e2eb4e91a07 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:22:27 +0200 Subject: [PATCH 03/23] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 170adc1..1229c7e 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Currently documentation is available for the following systems: * [HEBBE](docs/hebbe.md) * [ICR_DAVROS](docs/icr_davros.md) * [KRAKEN](docs/kraken.md) +* [MPCDF](docs/mpcdf.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) From 0a685c1cc2568fe06962e180e4c31f48defb7efa Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:24:11 +0200 Subject: [PATCH 04/23] Update mpcdf.config --- conf/mpcdf.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 9a109b0..849ba9c 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -1,5 +1,5 @@ params { - config_profile_description = 'MPCDF HPC profiles provided by nf-core/configs.' + config_profile_description = 'MPCDF HPC profiles (unoffically) provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://www.mpcdf.mpg.de/services/computing' } @@ -17,7 +17,7 @@ profiles { } executor { - queueSize =8 + queueSize = 8 } params { From 09cd5cad6565e99cc872064d63a93e8362ffd35b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:25:37 +0200 Subject: [PATCH 05/23] Update mpcdf.md --- docs/mpcdf.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 11d4474..89712d0 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -12,11 +12,18 @@ Currently the following clusters are supported: cobra, draco* (* coming soon) # cobra -Cobra does not currently support singularity, therefore the anaconda/module should be loaded for use. +Cobra does not currently support singularity, therefore the anaconda/module is loaded for each process. Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). To use: `-profile cobra,mpcdf` +Sets the following parameters: + +Maximum parallel running jobs: 8 +Max. memory: 750.GB +Max. CPUs: 80 +Max. walltime: 24.h + # draco From 9df85be081f386c19e303d1529df39aa816053f1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Sep 2020 20:27:15 +0200 Subject: [PATCH 06/23] Update mpcdf.md --- docs/mpcdf.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 89712d0..876dd8b 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -6,11 +6,11 @@ All nf-core pipelines have been successfully configured for use on the HPCs at [ To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. -Currently the following clusters are supported: cobra, draco* (* coming soon) +Currently the following clusters are supported: cobra, draco\* (\* coming soon) >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). -# cobra +## cobra Cobra does not currently support singularity, therefore the anaconda/module is loaded for each process. @@ -25,5 +25,4 @@ Max. memory: 750.GB Max. CPUs: 80 Max. walltime: 24.h -# draco - +## draco From 65fce5c2a539df834809eaf11d444bc545156694 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:52:46 +0200 Subject: [PATCH 07/23] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index c856521..cebbd52 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,6 +30,7 @@ profiles { gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } + mpcdf { includeConfig "${params.custom_config_base}/conf/mpcdf.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } From 5c7ab234b6d4d758dffbe435f6f972f94ed1c028 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:55:40 +0200 Subject: [PATCH 08/23] Create mpcdf.config --- conf/pipeline/eager/mpcdf.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 conf/pipeline/eager/mpcdf.config diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/conf/pipeline/eager/mpcdf.config @@ -0,0 +1 @@ + From b4e275c3b99db0dd3d0b3fcacba605bd72d48939 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:57:25 +0200 Subject: [PATCH 09/23] Add MPCDF for EAGER --- conf/pipeline/eager/mpcdf.config | 60 ++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 8b13789..5c26a7a 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -1 +1,61 @@ +// Profile config names for nf-core/configs +params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager MPCDF profile provided by nf-core/configs' +} + +// Specific nf-core/eager process configuration +process { + + withName: malt { + maxRetries = 1 + memory = 750.gb + cpus = 80 + time = 24.h + } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = 24.h + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = 24.h + } + +} From 4f991d2c0c47574fd7755e617c41571850ba1485 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 19:59:27 +0200 Subject: [PATCH 10/23] Update eager.config --- pipeline/eager.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipeline/eager.config b/pipeline/eager.config index 36f7cae..9242ecb 100644 --- a/pipeline/eager.config +++ b/pipeline/eager.config @@ -10,4 +10,6 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/pipeline/eager/shh.config" } -} \ No newline at end of file + mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } + +} From af8d25ed0922fc3b921a5e26205b57d86529856e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 20:14:35 +0200 Subject: [PATCH 11/23] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 019a3ee..2d3f9c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 4335e5deb7c2d4a38858ed15f393a2af48b6516e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Sep 2020 20:17:09 +0200 Subject: [PATCH 12/23] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 5c26a7a..ead449d 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -6,6 +6,8 @@ params { config_profile_description = 'nf-core/eager MPCDF profile provided by nf-core/configs' } +profile { +cobra { // Specific nf-core/eager process configuration process { @@ -57,5 +59,6 @@ process { memory = { check_max( 256.GB * task.attempt, 'memory' ) } time = 24.h } - + } + } } From 3b7902d11b29ef95de8877001104153229377d8a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 6 Sep 2020 08:23:35 +0200 Subject: [PATCH 13/23] Mad multi-core increase CPUs on retry --- conf/pipeline/eager/mpcdf.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index ead449d..832e5ed 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -37,25 +37,25 @@ process { } withLabel:'mc_small'{ - cpus = { check_max( 2, 'cpus' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_medium' { - cpus = { check_max( 4, 'cpus' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = 24.h } withLabel:'mc_huge'{ - cpus = { check_max( 32, 'cpus' ) } + cpus = { check_max( 32 * task.attempt, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } time = 24.h } From bf87ec24e7bfe6b654b7060cb33c71e5d37917b8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 17:45:38 +0200 Subject: [PATCH 14/23] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 832e5ed..046b3dc 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -13,7 +13,7 @@ process { withName: malt { maxRetries = 1 - memory = 750.gb + memory = 750.GB cpus = 80 time = 24.h } From c6f7573d01c138ada32e0efa2bccbb5082d74810 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 17:48:54 +0200 Subject: [PATCH 15/23] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 046b3dc..796bad9 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -13,7 +13,7 @@ process { withName: malt { maxRetries = 1 - memory = 750.GB + memory = 725.GB cpus = 80 time = 24.h } From 3115963b3e0575e90136beb674c68eb4fa465c82 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 15 Sep 2020 19:53:27 +0200 Subject: [PATCH 16/23] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 849ba9c..dafb362 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -22,7 +22,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' - max_memory = 750.GB + max_memory = 725.GB max_cpus = 80 max_time = 24.h } From 5b3492d24064535eef70e05d75a7643c494950f3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:26 +0200 Subject: [PATCH 17/23] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index dafb362..7d51680 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -23,7 +23,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' max_memory = 725.GB - max_cpus = 80 + max_cpus = 40 max_time = 24.h } } From bbbdd3cb0fc8b97321b012453ac5cf8124c213ef Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:42 +0200 Subject: [PATCH 18/23] Update mpcdf.config --- conf/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 7d51680..dafb362 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -23,7 +23,7 @@ profiles { params { config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' max_memory = 725.GB - max_cpus = 40 + max_cpus = 80 max_time = 24.h } } From 11f1fb71db7bffbfe7740c4b9bcc1c2dbad5f6df Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 16 Sep 2020 10:21:58 +0200 Subject: [PATCH 19/23] Update mpcdf.config --- conf/pipeline/eager/mpcdf.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/mpcdf.config b/conf/pipeline/eager/mpcdf.config index 796bad9..1b08e73 100644 --- a/conf/pipeline/eager/mpcdf.config +++ b/conf/pipeline/eager/mpcdf.config @@ -14,7 +14,7 @@ process { withName: malt { maxRetries = 1 memory = 725.GB - cpus = 80 + cpus = 40 time = 24.h } From 05deeca2d6e5cb4f2e9bfadf4ddf23b0bafca829 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Oct 2020 14:22:35 +0200 Subject: [PATCH 20/23] Update mpcdf.config --- conf/mpcdf.config | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index dafb362..8a0eed2 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -9,7 +9,7 @@ profiles { cobra { // Does not have singularity! Conda module must be used, but it is // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} - // To create common cache dir (only until /u/ periodically cleared!) + // To create common cache dir process { beforeScript = 'module load anaconda/3/2020.02' @@ -21,10 +21,33 @@ profiles { } params { - config_profile_description = 'MPCDF cobra profile provided by nf-core/configs.' + config_profile_description = 'MPCDF cobra profile (unofficially) provided by nf-core/configs.' max_memory = 725.GB max_cpus = 80 max_time = 24.h } } + + raven { + // Does not have singularity! Conda module must be used, but it is + // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} + // to create common cache dir + + process { + beforeScript = 'module load anaconda/3/2020.02' + executor = 'slurm' + } + + executor { + queueSize = 8 + } + + params { + config_profile_description = 'MPCDF raven profile (unofficially) provided by nf-core/configs.' + max_memory = 368.GB + max_cpus = 192 + max_time = 24.h + } + } + } From 0d6e269322bd4de0e0d2cdc39346023e9bfb6a41 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Oct 2020 14:24:08 +0200 Subject: [PATCH 21/23] Update mpcdf.md --- docs/mpcdf.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 876dd8b..52f0a71 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -26,3 +26,20 @@ Max. CPUs: 80 Max. walltime: 24.h ## draco + +:hammer_and_wrench: under testing. + +## raven + +Raven does not currently support singularity, therefore the anaconda/module is loaded for each process. + +Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). + +To use: `-profile raven,mpcdf` + +Sets the following parameters: + +Maximum parallel running jobs: 8 +Max. memory: 368.GB +Max. CPUs: 192 +Max. walltime: 24.h From 3995057d7f8e83c454e25b0e535cc22e45fa1ee6 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Oct 2020 10:42:46 +0200 Subject: [PATCH 22/23] Update mpcdf.config --- conf/mpcdf.config | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 8a0eed2..481ebc4 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -27,8 +27,7 @@ profiles { max_time = 24.h } } - - raven { + raven { // Does not have singularity! Conda module must be used, but it is // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} // to create common cache dir @@ -49,5 +48,4 @@ profiles { max_time = 24.h } } - -} +} From 7088ed196fac71055ce9504c6ec79a05cef6ad94 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Oct 2020 11:08:29 +0200 Subject: [PATCH 23/23] Update mpcdf.md --- docs/mpcdf.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 52f0a71..af16d76 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -6,7 +6,7 @@ All nf-core pipelines have been successfully configured for use on the HPCs at [ To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. -Currently the following clusters are supported: cobra, draco\* (\* coming soon) +Currently the following clusters are supported: cobra, raven >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). @@ -20,10 +20,10 @@ To use: `-profile cobra,mpcdf` Sets the following parameters: -Maximum parallel running jobs: 8 -Max. memory: 750.GB -Max. CPUs: 80 -Max. walltime: 24.h +- Maximum parallel running jobs: 8 +- Max. memory: 750.GB +- Max. CPUs: 80 +- Max. walltime: 24.h ## draco @@ -31,7 +31,7 @@ Max. walltime: 24.h ## raven -Raven does not currently support singularity, therefore the anaconda/module is loaded for each process. +Raven does not currently support singularity, therefore `module load anaconda/3/2020.02` is loaded for each process. Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). @@ -39,7 +39,7 @@ To use: `-profile raven,mpcdf` Sets the following parameters: -Maximum parallel running jobs: 8 -Max. memory: 368.GB -Max. CPUs: 192 -Max. walltime: 24.h +- Maximum parallel running jobs: 8 +- Max. memory: 368.GB +- Max. CPUs: 192 +- Max. walltime: 24.h