From 54dafbcfa32cf6374f3c8c7c8559b6cf465667c4 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Thu, 5 Dec 2019 12:09:11 +0100 Subject: [PATCH 1/5] Condensing SHH profiles following https://github.com/nf-core/configs/pull/100 --- conf/{shh_sdag.config => shh.config} | 22 +++++++++++++++++--- conf/shh_cdag.config | 30 ---------------------------- docs/shh.md | 4 ++-- nfcore_custom.config | 3 +-- 4 files changed, 22 insertions(+), 37 deletions(-) rename conf/{shh_sdag.config => shh.config} (57%) delete mode 100644 conf/shh_cdag.config diff --git a/conf/shh_sdag.config b/conf/shh.config similarity index 57% rename from conf/shh_sdag.config rename to conf/shh.config index b865123..118e13b 100644 --- a/conf/shh_sdag.config +++ b/conf/shh.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'MPI-SHH SDAG cluster profile provided by nf-core/configs.' + config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' } @@ -22,9 +22,25 @@ executor { } params { - max_memory = 2.TB - max_cpus = 128 + max_memory = 256.GB + max_cpus = 32 max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" } + + +profiles { + cdag { + config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' + max_memory = 256.GB + max_cpus = 32 + max_time = 720.h + } + sdag { + config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' + max_memory = 2.TB + max_cpus = 128 + max_time = 720.h + } +} \ No newline at end of file diff --git a/conf/shh_cdag.config b/conf/shh_cdag.config deleted file mode 100644 index c0a75c0..0000000 --- a/conf/shh_cdag.config +++ /dev/null @@ -1,30 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'MPI-SHH CDAG cluster profile provided by nf-core/configs.' - config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' - config_profile_url = 'https://shh.mpg.de' -} - -singularity { - enabled = true - autoMounts = true - runOptions = '-B /run/shm:/run/shm' - cacheDir = "/projects1/singularity_scratch/cache/" -} - -process { - executor = 'slurm' - queue = { task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } -} - -executor { - queueSize = 16 -} - -params { - max_memory = 256.GB - max_cpus = 32 - max_time = 720.h - //Illumina iGenomes reference file path - igenomes_base = "/projects1/public_data/igenomes/" -} diff --git a/docs/shh.md b/docs/shh.md index 02b3ca7..c780f21 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de). -To use, run the pipeline either with `-profile shh_sdag` or `-profile ssh_cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: +To use, run the pipeline with `-profile ssh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile ssh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: ```bash /projects1/singularity_scratch/cache/ @@ -10,7 +10,7 @@ To use, run the pipeline either with `-profile shh_sdag` or `-profile ssh_cdag`. however this will likely change to a read-only directory in the future that will be managed by the IT team. -This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `ssh_sdag` additionally allows for submission of jobs to the `supercruncher` when a job's requested memory exceeds 756GB. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile ssh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team. >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 32a01f4..90145b3 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -35,8 +35,7 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - shh_sdag { includeConfig "${params.custom_config_base}/conf/shh_sdag.config" } - shh_cdag { includeConfig "${params.custom_config_base}/conf/shh_cdag.config" } + shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; From 41a795f996dff9f970135dec3ddfaa1e84a443a9 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Thu, 5 Dec 2019 12:14:01 +0100 Subject: [PATCH 2/5] Fixed github actions test --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7d78a45..60e3b7e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,8 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh_sdag', 'shh_cdag', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', +'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 3fc1377ff273ffe22faf7a8ad13e41058d871037 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Thu, 5 Dec 2019 12:16:01 +0100 Subject: [PATCH 3/5] Remove mysterious new line --- .github/workflows/main.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 60e3b7e..fd68025 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: Check whether profiles are all tested - run: | + run: | python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest @@ -16,8 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', -'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 678a3841b6a3ee126d4f8d0f68549b03909c1fb8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 5 Dec 2019 12:23:06 +0100 Subject: [PATCH 4/5] Update conf/shh.config Co-Authored-By: Maxime Garcia --- conf/shh.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 118e13b..6ad3b2f 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -33,7 +33,6 @@ params { profiles { cdag { config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' - max_memory = 256.GB max_cpus = 32 max_time = 720.h } @@ -43,4 +42,4 @@ profiles { max_cpus = 128 max_time = 720.h } -} \ No newline at end of file +} From 842642a2bb562e39dd484c2f311a0e590754699e Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 5 Dec 2019 12:23:36 +0100 Subject: [PATCH 5/5] Apply suggestions from code review Co-Authored-By: Maxime Garcia --- conf/shh.config | 3 --- 1 file changed, 3 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 6ad3b2f..190caad 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -33,13 +33,10 @@ params { profiles { cdag { config_profile_description = 'MPI-SHH CDAG profile, provided by nf-core/configs.' - max_cpus = 32 - max_time = 720.h } sdag { config_profile_description = 'MPI-SHH SDAG profile, provided by nf-core/configs.' max_memory = 2.TB max_cpus = 128 - max_time = 720.h } }