From 7d8770af1a1ad08fdaa9d83bde33e1e541969de2 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 16:39:55 +0100 Subject: [PATCH 01/10] Create sanger.config --- conf/sanger.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 conf/sanger.config diff --git a/conf/sanger.config b/conf/sanger.config new file mode 100644 index 0000000..686a3af --- /dev/null +++ b/conf/sanger.config @@ -0,0 +1,26 @@ +params { + config_profile_description = 'The Wellcome Sanger Institute HPC cluster profile' + config_profile_contact = 'Anthony Underwood (@aunderwo)' + config_profile_url = 'https://www.sanger.ac.uk/group/informatics-support-group/' +} + +singularity { + enabled = true + cacheDir = "${baseDir}/singularity" + runOptions = '--bind /lustre --bind /nfs/pathnfs01 --bind /nfs/pathnfs02 --bind /nfs/pathnfs03 --bind /nfs/pathnfs04 --bind /nfs/pathnfs05 --bind /nfs/pathnfs06 --no-home' +} + +process{ + executor = 'lsf' + queue = 'normal' + errorStrategy = { task.attempt <= 5 ? "retry" : "finish" } + process.maxRetries = 5 +} + +executor{ + name = 'lsf' + perJobMemLimit = true + poolSize = 4 + submitRateLimit = '5 sec' + killBatchSize = 50 +} From 7163e8723d7f7011bfb3f6aa225c170848a6c5c1 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:17:42 +0100 Subject: [PATCH 02/10] Create sanger.md --- docs/sanger.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/sanger.md diff --git a/docs/sanger.md b/docs/sanger.md new file mode 100644 index 0000000..be39bfa --- /dev/null +++ b/docs/sanger.md @@ -0,0 +1,47 @@ +# nf-core/configs: Wellcome Sanger Institute Configuration + +To use, run the pipeline with `-profile sanger`. This will download and launch the [`sanger.config`](../conf/sanger.config) which has been +pre-configured with a setup suitable for the Wellcome Sanger Institute LSF cluster. +Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or +a Singularity image downloaded directly before execution of the pipeline. + +## Running the workflow on the Wellcome Sanger Institute cluster + +The latest version of Nextflow is not installed by default on the cluster. You will need to install it into a directory you have write access to + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. +Nextflow shouldn't run directly on the submission node but on a compute node. +To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` + +``` +#!/bin/bash +#BSUB -o /path/to/a/log/dir/%J.o +#BSUB -e /path/to/a/log/dir//%J.e +#BSUB -M 8000 +#BSUB -q long +#BSUB -n 4 + +export HTTP_PROXY='http://wwwcache.sanger.ac.uk:3128' +export HTTPS_PROXY='http://wwwcache.sanger.ac.uk:3128' +export NXF_ANSI_LOG=false +export NXF_OPTS="-Xms8G -Xmx8G -Dnxf.pool.maxThreads=2000" +export NXF_VER=21.03.0-edge + + +/path/to/your/install/nextflow run \ +/path/to/nf-core/pipeline/main.nf \ +-w /path/to/some/dir/work \ +-profile sanger \ +-c my_specific.config \ +-qs 1000 \ +-resume + +## clean up on exit 0 - delete this if you want to keep the work dir +status=$? +if [[ $status -eq 0 ]]; then + rm -r /path/to/some/dir/work +fi +``` + From 40eb3e02010af4653b44748116102eb6f6b0f376 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:20:30 +0100 Subject: [PATCH 03/10] Update sanger.md --- docs/sanger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index be39bfa..f1e5c29 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -2,7 +2,7 @@ To use, run the pipeline with `-profile sanger`. This will download and launch the [`sanger.config`](../conf/sanger.config) which has been pre-configured with a setup suitable for the Wellcome Sanger Institute LSF cluster. -Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or +Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or a Singularity image downloaded directly before execution of the pipeline. ## Running the workflow on the Wellcome Sanger Institute cluster From cc18d213dc44742ac579b6e7aa6498656e6c7313 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:24:08 +0100 Subject: [PATCH 04/10] Update sanger.md --- docs/sanger.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sanger.md b/docs/sanger.md index f1e5c29..bb0e607 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -15,7 +15,7 @@ Nextflow manages each process as a separate job that is submitted to the cluster Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` -``` +```bash #!/bin/bash #BSUB -o /path/to/a/log/dir/%J.o #BSUB -e /path/to/a/log/dir//%J.e @@ -44,4 +44,3 @@ if [[ $status -eq 0 ]]; then rm -r /path/to/some/dir/work fi ``` - From 6175362049ab8a378df5464970b23c6f40b00c1c Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:47:48 +0100 Subject: [PATCH 05/10] Add sanger profile --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 88d7a62..9b4fbda 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -43,6 +43,7 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } + sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } From fa267c0cf95a5657d33ce07892b073b8a0629d57 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 21:01:25 +0100 Subject: [PATCH 06/10] Add sanger to tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd2170c..65cabe5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'sanger', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 351c5773f0541dd445649e6c4af1b8a8590e0a5e Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 11:59:34 +0100 Subject: [PATCH 07/10] Add queue limits for 'normal' --- conf/sanger.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/sanger.config b/conf/sanger.config index 686a3af..63952c8 100644 --- a/conf/sanger.config +++ b/conf/sanger.config @@ -24,3 +24,9 @@ executor{ submitRateLimit = '5 sec' killBatchSize = 50 } + +params { + max_memory = 128.GB + max_cpus = 64 + max_time = 12.h +} From b16b011bcc2fc72ad7b3b84b410d895cae139cdf Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 12:02:11 +0100 Subject: [PATCH 08/10] Add ~/bin as recommended location for nextflow executable --- docs/sanger.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sanger.md b/docs/sanger.md index bb0e607..483683c 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -11,6 +11,8 @@ The latest version of Nextflow is not installed by default on the cluster. You w - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) +Recommendation place to move the `nextflow` executable is `~/bin` so that it's in the `PATH`. + Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` @@ -27,10 +29,10 @@ export HTTP_PROXY='http://wwwcache.sanger.ac.uk:3128' export HTTPS_PROXY='http://wwwcache.sanger.ac.uk:3128' export NXF_ANSI_LOG=false export NXF_OPTS="-Xms8G -Xmx8G -Dnxf.pool.maxThreads=2000" -export NXF_VER=21.03.0-edge +export NXF_VER=21.04.0-edge -/path/to/your/install/nextflow run \ +nextflow run \ /path/to/nf-core/pipeline/main.nf \ -w /path/to/some/dir/work \ -profile sanger \ From f414446a8845852e4fd8032fb3d45da385e33f10 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 15:38:36 +0100 Subject: [PATCH 09/10] Add info on how to load Singularity into your path --- docs/sanger.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index 483683c..940834e 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -11,9 +11,16 @@ The latest version of Nextflow is not installed by default on the cluster. You w - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) -Recommendation place to move the `nextflow` executable is `~/bin` so that it's in the `PATH`. +A recommended place to move the `nextflow` executable to is `~/bin` so that it's in the `PATH`. + Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. +Since the Nextflow pipeline will submit individual jobs for each process to the cluster and dependencies will be provided bu Singularity images you shoudl make sure that your account has access to the Singularity binary by adding these lines to your `.bashrc` file + +```bash +[[ -f /software/pathogen/farm5 ]] && module load ISG/singularity +``` + Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` From 3e8f9b3d4e60f912039f10192a11ee099e5c668a Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 15:57:09 +0100 Subject: [PATCH 10/10] Remove blank lines --- docs/sanger.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index 940834e..ee75755 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -13,7 +13,6 @@ The latest version of Nextflow is not installed by default on the cluster. You w A recommended place to move the `nextflow` executable to is `~/bin` so that it's in the `PATH`. - Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. Since the Nextflow pipeline will submit individual jobs for each process to the cluster and dependencies will be provided bu Singularity images you shoudl make sure that your account has access to the Singularity binary by adding these lines to your `.bashrc` file