From b240142cc327020832cdaafe1ccba6b5fd0b642c Mon Sep 17 00:00:00 2001 From: theoportlock Date: Wed, 8 Sep 2021 14:33:39 +0200 Subject: [PATCH 1/6] Adds rosalind configuration --- .github/workflows/main.yml | 1 + README.md | 1 + conf/rosalind.config | 28 ++++++++++++++++++++++++++++ docs/cambridge.md | 2 +- docs/rosalind.md | 23 +++++++++++++++++++++++ nfcore_custom.config | 1 + 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 conf/rosalind.config create mode 100644 docs/rosalind.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c5e6a57..f1f658d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,7 @@ jobs: - 'pasteur' - 'phoenix' - 'prince' + - 'rosalind' - 'sanger' - 'seg_globe' - 'shh' diff --git a/README.md b/README.md index 29fbce2..dc50e8a 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Currently documentation is available for the following systems: * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) +* [ROSALIND](docs/rosalind.md) * [SANGER](docs/sanger.md) * [SEG_GLOBE](docs/seg_globe.md) * [SHH](docs/shh.md) diff --git a/conf/rosalind.config b/conf/rosalind.config new file mode 100644 index 0000000..bb967c3 --- /dev/null +++ b/conf/rosalind.config @@ -0,0 +1,28 @@ +params { + config_profile_description = 'Kings College London Rosalind HPC' + config_profile_contact = 'Theo Portlock' + config_profile_url = 'https://www.rosalind.kcl.ac.uk/' +} + +singularity { + enabled = true + autoMounts = true + docker.enabled = false +} + +params { + max_memory = 64.GB + max_cpus = 16 + max_time = 24.h + partition = 'shared' +} + +process { + executor = 'slurm' + maxRetries = 3 + clusterOptions = { "--partition=$params.partition ${params.clusterOptions ?: ''}" } +} + +executor { + submitRateLimit = '1 sec' +} diff --git a/docs/cambridge.md b/docs/cambridge.md index ce833d7..4f18088 100644 --- a/docs/cambridge.md +++ b/docs/cambridge.md @@ -1,7 +1,7 @@ # nf-core/configs: Cambridge HPC Configuration All nf-core pipelines have been successfully configured for use on the Cambridge HPC cluster at the [The University of Cambridge](https://www.cam.ac.uk/). -To use, run the pipeline with `-profile cambridge`. This will download and launch the [`cambridge.config`](../conf/cambridge.config) whichhas been pre-configured +To use, run the pipeline with `-profile cambridge`. This will download and launch the [`cambridge.config`](../conf/cambridge.config) which has been pre-configured with a setup suitable for the Cambridge HPC cluster. Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or a Singularity image downloaded directly before execution of the pipeline. diff --git a/docs/rosalind.md b/docs/rosalind.md new file mode 100644 index 0000000..6828c87 --- /dev/null +++ b/docs/rosalind.md @@ -0,0 +1,23 @@ +# nf-core/configs: Rosalind Configuration + +All nf-core pipelines have been successfully configured for use on the Rosalind CLuster at [Kings College London](https://rosalind.kcl.ac.uk/). +To use, run the pipeline with `-profile rosalind`. This will download and launch the [`rosalind.config`](../conf/rosalind.config) which has been pre-configured with a setup suitable for the rosalind cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Using Nextflow on Rosalind + +Before running the pipeline you will need to configure Nextflow and Singularity. There is no Nextflow module on Rosalind at this time. This can be done with the following commands: + +```bash +## Load Singularity environment modules - these commands can be placed in your ~/.bashrc also +module load apps/openjdk +module load apps/singularity + +## Download Nextflow-all +wget https://github.com/nextflow-io/nextflow/releases/download/v21.04.3/nextflow-21.04.3-all +chmod a+x nextflow-21.04.3-all +mv nextflow-21.04.3-all ~/bin/nextflow +``` + +## Additional information + +The default shared partition resource limits are defined as ten percent of the total available to the cluster at any one point in time. The limitations defined by this configuration are conservative and are projected to be increased as greater computational resources are introduced in the near future. diff --git a/nfcore_custom.config b/nfcore_custom.config index 74d25b9..19a2bb1 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -46,6 +46,7 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } + rosalind { includeConfig "${params.custom_config_base}/conf/rosalind.config" } sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} shh { includeConfig "${params.custom_config_base}/conf/shh.config" } From 05ad3e0577ca8f816eb83f7fb518808165433e23 Mon Sep 17 00:00:00 2001 From: Theo Portlock <33433082+theoportlock@users.noreply.github.com> Date: Thu, 9 Sep 2021 13:37:02 +0200 Subject: [PATCH 2/6] Update conf/rosalind.config Co-authored-by: James A. Fellows Yates --- conf/rosalind.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/rosalind.config b/conf/rosalind.config index bb967c3..2b2e389 100644 --- a/conf/rosalind.config +++ b/conf/rosalind.config @@ -1,7 +1,7 @@ params { config_profile_description = 'Kings College London Rosalind HPC' config_profile_contact = 'Theo Portlock' - config_profile_url = 'https://www.rosalind.kcl.ac.uk/' + config_profile_url = 'https://rosalind.kcl.ac.uk/' } singularity { From 3c86f8f47ced7d9484d36c345e3d812f7d96f074 Mon Sep 17 00:00:00 2001 From: theoportlock Date: Wed, 15 Sep 2021 11:34:41 +0200 Subject: [PATCH 3/6] Removes the broken clusterOptions --- conf/rosalind.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/rosalind.config b/conf/rosalind.config index bb967c3..e3de272 100644 --- a/conf/rosalind.config +++ b/conf/rosalind.config @@ -20,7 +20,7 @@ params { process { executor = 'slurm' maxRetries = 3 - clusterOptions = { "--partition=$params.partition ${params.clusterOptions ?: ''}" } + clusterOptions = { "--partition=$params.partition" } } executor { From 9116a09afe277c90cb753e886df6beffad1f0e79 Mon Sep 17 00:00:00 2001 From: theoportlock Date: Fri, 17 Sep 2021 11:48:22 +0200 Subject: [PATCH 4/6] Fixes formatting and ignore_params --- conf/rosalind.config | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/conf/rosalind.config b/conf/rosalind.config index 8173633..070cf11 100644 --- a/conf/rosalind.config +++ b/conf/rosalind.config @@ -1,28 +1,29 @@ params { - config_profile_description = 'Kings College London Rosalind HPC' - config_profile_contact = 'Theo Portlock' - config_profile_url = 'https://rosalind.kcl.ac.uk/' + config_profile_description = 'Kings College London Rosalind HPC' + config_profile_contact = 'Theo Portlock' + config_profile_url = 'https://www.rosalind.kcl.ac.uk/' } singularity { - enabled = true - autoMounts = true - docker.enabled = false + enabled = true + autoMounts = true + docker.enabled = false } params { - max_memory = 64.GB - max_cpus = 16 - max_time = 24.h - partition = 'shared' + max_memory = 64.GB + max_cpus = 16 + max_time = 24.h + partition = 'shared' + schema_ignore_params = 'partition' } process { - executor = 'slurm' - maxRetries = 3 - clusterOptions = { "--partition=$params.partition" } + executor = 'slurm' + maxRetries = 3 + clusterOptions = { "--partition=$params.partition" } } executor { - submitRateLimit = '1 sec' + submitRateLimit = '1 sec' } From 0fd5265f7bcdf090b352a87eabd3a0a06ce39733 Mon Sep 17 00:00:00 2001 From: theoportlock Date: Fri, 17 Sep 2021 13:21:38 +0200 Subject: [PATCH 5/6] Updates the README --- docs/rosalind.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/rosalind.md b/docs/rosalind.md index 6828c87..3226449 100644 --- a/docs/rosalind.md +++ b/docs/rosalind.md @@ -18,6 +18,8 @@ chmod a+x nextflow-21.04.3-all mv nextflow-21.04.3-all ~/bin/nextflow ``` +By default, the shared partition is used for job submission. Other partitions can be specified using the `--partition ` argument to the run. + ## Additional information The default shared partition resource limits are defined as ten percent of the total available to the cluster at any one point in time. The limitations defined by this configuration are conservative and are projected to be increased as greater computational resources are introduced in the near future. From 3b95fd9ee39061d21f193adc81e806cff9bbe354 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Fri, 17 Sep 2021 13:15:09 +0100 Subject: [PATCH 6/6] Update conf/rosalind.config --- conf/rosalind.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/rosalind.config b/conf/rosalind.config index 070cf11..05e3af7 100644 --- a/conf/rosalind.config +++ b/conf/rosalind.config @@ -15,7 +15,7 @@ params { max_cpus = 16 max_time = 24.h partition = 'shared' - schema_ignore_params = 'partition' + schema_ignore_params = 'partition,genomes,modules' } process {