From 0acdcc381387338ce35bbf42a89f44cac3cb6f92 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sat, 4 May 2019 23:20:15 +0200 Subject: [PATCH 01/26] Add Aquila Config Singapore --- README.md | 1 + conf/aquila.config | 26 ++++++++++++++++++++++++++ docs/aquila.md | 8 ++++++++ nfcore_custom.config | 1 + 4 files changed, 36 insertions(+) create mode 100644 conf/aquila.config create mode 100644 docs/aquila.md diff --git a/README.md b/README.md index eadd3bb..d16b13e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: +* [AQUILA](docs/aquila.md) * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) diff --git a/conf/aquila.config b/conf/aquila.config new file mode 100644 index 0000000..05035d4 --- /dev/null +++ b/conf/aquila.config @@ -0,0 +1,26 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'A*STAR Aquila cluster profile provided by nf-core/configs.' + config_profile_contact = 'Andreas Wilm (@andreas-wilm)' + config_profile_url = 'https://www.a-star.edu.sg/' +} + +singularity { + enabled = true +} + +process { + beforeScript = { 'source /mnt/projects/rpd/rc/init.2017-04; module load miniconda3; set +u; source activate ' + params.container.replaceAll(/[\/:]/, '-').replaceAll(/nfcore/, 'nf-core') + "; set -u;" } + executor = 'sge' + clusterOptions = { "-l mem_free=" + task.memory.toString().replaceAll(/[\sB]/,'') } + penv = 'OpenMP' +} + +params { + igenomes_base = '/nfsmounts/igenomes' + max_memory = 128.GB + max_cpus = 28 + max_time = 48.h +} + + diff --git a/docs/aquila.md b/docs/aquila.md new file mode 100644 index 0000000..3e0575e --- /dev/null +++ b/docs/aquila.md @@ -0,0 +1,8 @@ +# nf-core/configs: Aquila Configuration + +All nf-core pipelines have been successfully configured for use on the Aquila cluster at A*STAR Singapore. + +To use, run the pipeline with `-profile aquila`. This will download and launch the [`aquila.config`](../conf/aquila.config) which has been pre-configured with a setup suitable for the Aquila cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +>NB: You will need an account to use the HPC cluster Aquila in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 7374fa0..17f51c5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From 29a141adca47d2b7908b6d29b0d0eccfe051a2be Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 16:54:58 +0200 Subject: [PATCH 02/26] Sort it --- README.md | 2 +- nfcore_custom.config | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4b4d41..af4aad1 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Currently documentation is available for the following clusters: * [HEBBE](docs/hebbe.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) +* [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) @@ -100,7 +101,6 @@ Currently documentation is available for the following clusters: * [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) * [UZH](docs/uzh.md) -* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` diff --git a/nfcore_custom.config b/nfcore_custom.config index ec40134..469d6af 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -13,6 +13,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } + bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } @@ -27,15 +28,14 @@ profiles { hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } - pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From 4e48d4cc62b8f4b7415867c2377534f0e9a17c74 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:00 +0200 Subject: [PATCH 03/26] Fix remaining points by drpatelh --- README.md | 4 ++-- docs/pasteur.md | 34 ++++++++++++++++++---------------- nfcore_custom.config | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index af4aad1..5323165 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: * [AQUILA](docs/aquila.md) -* [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) +* [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) @@ -98,8 +98,8 @@ Currently documentation is available for the following clusters: * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) +* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` diff --git a/docs/pasteur.md b/docs/pasteur.md index 554761b..6e2d825 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,36 +4,38 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - - ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. -- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. The compute nodes don't have access to internet so you need to run it offline. To do that: + 1. Create a virtualenv to install nf-core -```bash -module purge -module load Python/3.6.0 -module load java -module load singularity -cd /path/to/nf-core/workflows -virtualenv .venv -p python3 -. .venv/bin/activate -``` + + ```bash + module purge + module load Python/3.6.0 + module load java + module load singularity + cd /path/to/nf-core/workflows + virtualenv .venv -p python3 + . .venv/bin/activate + ``` + 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) -4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) -5. Run nextflow on a compute node: +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/configs#offline-usage) +5. Run nextflow on a compute node: + ```bash # create a terminal -tmux +tmux # Get a compute node salloc @@ -46,7 +48,7 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ diff --git a/nfcore_custom.config b/nfcore_custom.config index 469d6af..69bb8f3 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,8 +33,8 @@ profiles { prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 82ffedf895ad9d1afa28307fe406d64eb770844a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:36 +0200 Subject: [PATCH 04/26] Drop aquila --- docs/aquila.md | 8 -------- nfcore_custom.config | 1 - 2 files changed, 9 deletions(-) delete mode 100644 docs/aquila.md diff --git a/docs/aquila.md b/docs/aquila.md deleted file mode 100644 index 3e0575e..0000000 --- a/docs/aquila.md +++ /dev/null @@ -1,8 +0,0 @@ -# nf-core/configs: Aquila Configuration - -All nf-core pipelines have been successfully configured for use on the Aquila cluster at A*STAR Singapore. - -To use, run the pipeline with `-profile aquila`. This will download and launch the [`aquila.config`](../conf/aquila.config) which has been pre-configured with a setup suitable for the Aquila cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - ->NB: You will need an account to use the HPC cluster Aquila in order to run the pipeline. If in doubt contact IT. ->NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 69bb8f3..259b048 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,7 +12,6 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { - aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } From d9930b053c3cd91c3cfa20c0bb556cf3cd460346 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:45 +0200 Subject: [PATCH 05/26] Drop aquila --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5323165..c32df8c 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: -* [AQUILA](docs/aquila.md) * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) From ead80fb9d096820324a836cf4b1bafa0ebe2b41c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:54:38 +0200 Subject: [PATCH 06/26] Drop config --- conf/aquila.config | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 conf/aquila.config diff --git a/conf/aquila.config b/conf/aquila.config deleted file mode 100644 index 05035d4..0000000 --- a/conf/aquila.config +++ /dev/null @@ -1,26 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'A*STAR Aquila cluster profile provided by nf-core/configs.' - config_profile_contact = 'Andreas Wilm (@andreas-wilm)' - config_profile_url = 'https://www.a-star.edu.sg/' -} - -singularity { - enabled = true -} - -process { - beforeScript = { 'source /mnt/projects/rpd/rc/init.2017-04; module load miniconda3; set +u; source activate ' + params.container.replaceAll(/[\/:]/, '-').replaceAll(/nfcore/, 'nf-core') + "; set -u;" } - executor = 'sge' - clusterOptions = { "-l mem_free=" + task.memory.toString().replaceAll(/[\sB]/,'') } - penv = 'OpenMP' -} - -params { - igenomes_base = '/nfsmounts/igenomes' - max_memory = 128.GB - max_cpus = 28 - max_time = 48.h -} - - From 5e54912ada55fa05cdca645584a185ed465c6389 Mon Sep 17 00:00:00 2001 From: maxibor Date: Fri, 25 Oct 2019 14:08:02 +0200 Subject: [PATCH 07/26] add queue choice --- conf/shh.config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 5cb3396..c23d331 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -14,7 +14,14 @@ singularity { process { executor = 'slurm' - queue = 'short' + switch(task.time){ + case {it < 2.h}: + queue = 'short' + case {it > 2.h && it < 48.h}: + queue = 'medium' + case {it > 48.h}: + queue = 'long' + } } executor { From 7a2fe44b43861a89755972048dd944cdacf28484 Mon Sep 17 00:00:00 2001 From: Maxime Borry Date: Fri, 25 Oct 2019 15:50:49 +0200 Subject: [PATCH 08/26] resource based queuing --- conf/shh.config | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index c23d331..14da627 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -14,14 +14,7 @@ singularity { process { executor = 'slurm' - switch(task.time){ - case {it < 2.h}: - queue = 'short' - case {it > 2.h && it < 48.h}: - queue = 'medium' - case {it > 48.h}: - queue = 'long' - } + queue = { task.memory > 756.GB ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } executor { @@ -29,9 +22,9 @@ executor { } params { - max_memory = 256.GB + max_memory = 2.TB max_cpus = 32 - max_time = 2.h + max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" } \ No newline at end of file From 5ec19e3abb25ffa47f09f9123057b31a50108ff9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 27 Oct 2019 13:28:00 +0100 Subject: [PATCH 09/26] Binac SMP / Long / Short Queue Additions --- conf/binac.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/binac.config b/conf/binac.config index 9447ca4..8047bd0 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,6 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { From 123f199b68ff287afacb2e6a3d3fb3078c3ae81f Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 27 Oct 2019 20:44:08 +0100 Subject: [PATCH 10/26] cbe: dynamic qos logic --- conf/cbe.config | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index c303f92..aed9866 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -8,25 +8,15 @@ params { process { executor = 'slurm' module = 'singularity/3.2.1' - queue = 'c' + queue = { task.memory <= 170.GB ? 'c' : 'm' } + clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } } singularity.enabled = true params { - target_qos = 'medium' - params.max_cpus = 36 - params.max_memory = 170.GB - igenomesIgnore = true -} - -if (params.target_qos == 'short') { - params.max_time = 8.h - process.clusterOptions = '--qos short' -} else if (params.target_qos == 'medium') { - params.max_time = 2.d - process.clusterOptions = '--qos medium' -} else { params.max_time = 14.d - process.clusterOptions = '--qos long' -} + params.max_cpus = 36 + params.max_memory = 4.TB + igenomesIgnore = true +} \ No newline at end of file From 18ed48f39839d42e839566f379a1b9e6680b3d3b Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 28 Oct 2019 11:38:24 +0100 Subject: [PATCH 11/26] deprecate igenomesIgnore in favor of igenomes_ignore keep both for now until all the pipelines have been synced --- conf/cbe.config | 3 ++- conf/czbiohub_aws.config | 3 ++- conf/genouest.config | 3 ++- conf/hebbe.config | 3 ++- conf/mendel.config | 3 ++- conf/pasteur.config | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index aed9866..6e47361 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -18,5 +18,6 @@ params { params.max_time = 14.d params.max_cpus = 36 params.max_memory = 4.TB - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated } \ No newline at end of file diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 1dd8da2..98aebe8 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -55,7 +55,8 @@ params { awsregion = "us-west-2" awsqueue = "nextflow" - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated fc_extra_attributes = 'gene_name' fc_group_features = 'gene_id' diff --git a/conf/genouest.config b/conf/genouest.config index 68d82a6..801b9fb 100644 --- a/conf/genouest.config +++ b/conf/genouest.config @@ -16,7 +16,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated max_memory = 750.GB max_cpus = 80 max_time = 336.h diff --git a/conf/hebbe.config b/conf/hebbe.config index 500ec31..26296d5 100644 --- a/conf/hebbe.config +++ b/conf/hebbe.config @@ -18,7 +18,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated saveReference = true max_memory = 64.GB max_cpus = 20 diff --git a/conf/mendel.config b/conf/mendel.config index e658c87..0fb7e81 100644 --- a/conf/mendel.config +++ b/conf/mendel.config @@ -19,5 +19,6 @@ params { max_cpus = 32 max_memory = 128.GB max_time = 192.h - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated } diff --git a/conf/pasteur.config b/conf/pasteur.config index 1d95131..01d5227 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -16,7 +16,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated max_memory = 256.GB max_cpus = 28 max_time = 24.h From 83fcc0b2baf86d5ef4ca22206c940ce442901008 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:09:30 +0100 Subject: [PATCH 12/26] Adjust max memory --- conf/binac.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index 8047bd0..e2bce86 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,14 +13,14 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 128.GB + max_memory = 1024.GB max_cpus = 28 - max_time = 48.h + max_time = 168.h } weblog{ From 5348b8c609bac0b13003cc464f86ff6973ec1b95 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:12:31 +0100 Subject: [PATCH 13/26] Revert "BINAC SMP / Short / Long Updates" --- conf/binac.config | 5 ++--- docs/pasteur.md | 34 ++++++++++++++++------------------ 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index e2bce86..9447ca4 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,14 +13,13 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 1024.GB + max_memory = 128.GB max_cpus = 28 - max_time = 168.h + max_time = 48.h } weblog{ diff --git a/docs/pasteur.md b/docs/pasteur.md index 6e2d825..554761b 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,38 +4,36 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + + ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. - - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. The compute nodes don't have access to internet so you need to run it offline. To do that: - 1. Create a virtualenv to install nf-core - - ```bash - module purge - module load Python/3.6.0 - module load java - module load singularity - cd /path/to/nf-core/workflows - virtualenv .venv -p python3 - . .venv/bin/activate - ``` - +```bash +module purge +module load Python/3.6.0 +module load java +module load singularity +cd /path/to/nf-core/workflows +virtualenv .venv -p python3 +. .venv/bin/activate +``` 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) -4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/configs#offline-usage) -5. Run nextflow on a compute node: - +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) +5. Run nextflow on a compute node: ```bash # create a terminal -tmux +tmux # Get a compute node salloc @@ -48,7 +46,7 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ From 494aa8b53e9830a6f2d332e0d3ee40447a6dda8d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:15:54 +0100 Subject: [PATCH 14/26] Mini fix --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index e2bce86..cba2ab6 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,7 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { From 41e607e7f98c67cb181ceffa625002473997e0c2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:17:39 +0100 Subject: [PATCH 15/26] Fix expression --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index cba2ab6..748c9e6 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,7 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'short' : 'long'} } params { From e557ced1844e6bb6ce74e8f2b038abe2b7024819 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:21:41 +0100 Subject: [PATCH 16/26] Update max memory --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 748c9e6..f669793 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -18,7 +18,7 @@ process { params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 1024.GB + max_memory = 1000.GB max_cpus = 28 max_time = 168.h } From 5a9dbe5d7c5472f3001f5c860ce498b97b073ce4 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 28 Oct 2019 14:11:34 +0100 Subject: [PATCH 17/26] MENDEL is no more the cluster is no longer operational, was replaced by CBE --- README.md | 1 - conf/mendel.config | 24 ------------------------ docs/mendel.md | 18 ------------------ nfcore_custom.config | 1 - 4 files changed, 44 deletions(-) delete mode 100644 conf/mendel.config delete mode 100644 docs/mendel.md diff --git a/README.md b/README.md index 451c12f..20fafd7 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,6 @@ Currently documentation is available for the following clusters: * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) -* [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) diff --git a/conf/mendel.config b/conf/mendel.config deleted file mode 100644 index 0fb7e81..0000000 --- a/conf/mendel.config +++ /dev/null @@ -1,24 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'GMI MENDEL cluster profile provided by nf-core/configs' - config_profile_contact = 'Patrick Hüther (@phue)' - config_profile_url = 'http://www.gmi.oeaw.ac.at/' -} - -manifest { - nextflowVersion = '>=19.01.0' -} - -process { - beforeScript = 'module load Miniconda3/4.6.7' - executor = 'pbspro' - clusterOptions = { "-P $params.project" } -} - -params { - max_cpus = 32 - max_memory = 128.GB - max_time = 192.h - igenomes_ignore = true - igenomesIgnore = true //deprecated -} diff --git a/docs/mendel.md b/docs/mendel.md deleted file mode 100644 index ae0ad4c..0000000 --- a/docs/mendel.md +++ /dev/null @@ -1,18 +0,0 @@ -# nf-core/configs: MENDEL Configuration - -All nf-core pipelines have been successfully configured for use on the MENDEL CLUSTER at the Gregor Mendel Institute (GMI). - -To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be resolved via [bioconda](https://bioconda.github.io/). - -Before running the pipeline you will need to load Conda using the environment module system on MENDEL. You can do this by issuing the commands below: - -```bash -## Load Nextflow and Conda environment modules -module purge -module load Nextflow -module load Miniconda/4.6.7 -``` - ->NB: You will need an account to use the HPC cluster in order to run the pipeline. If in doubt contact the HPC team. - ->NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact the HPC team. diff --git a/nfcore_custom.config b/nfcore_custom.config index 06182f5..12a0c1e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -25,7 +25,6 @@ profiles { gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } - mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } From 47a7d57db4c7d2acec49f468d69eb340286d1d4c Mon Sep 17 00:00:00 2001 From: maxibor Date: Mon, 28 Oct 2019 16:09:00 +0100 Subject: [PATCH 18/26] update shh doc --- docs/shh.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/shh.md b/docs/shh.md index 67cf3b9..aff3f21 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -10,7 +10,8 @@ To use, run the pipeline with `-profile shh`. This will download and launch the however this will likely change to a read-only directory in the future that will be managed by IT. -Note that **the configuration file is currently optimised for `nf-core/eager`**. It will submit to the short queue but with a walltime of 2 hours. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. +Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. From 99f4d9b9bd1bdd7465e9453c0ff209654a0ed8a0 Mon Sep 17 00:00:00 2001 From: arontommi Date: Wed, 6 Nov 2019 13:10:09 +0100 Subject: [PATCH 19/26] documentation on how to fix bianca cluster configuration problem --- docs/uppmax.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/uppmax.md b/docs/uppmax.md index 07a7f14..d5bba37 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -26,6 +26,12 @@ If running on Bianca, you will have no internet connection and these configs wil Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. +Please note that Bianca only allocates 7 GB memory per core so max memory needs to be configured: + +```bash +--max_memory "112GB" +``` + ## Getting more memory If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: From 16a0ba9fd58db31ecd352140d45d70356250322e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 8 Nov 2019 10:06:23 +0100 Subject: [PATCH 20/26] Update cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index d5b1c3b..355f106 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -11,7 +11,7 @@ singularity { } process { - beforeScript = 'module load qbic/singularity_slurm/3.0.3' + beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' } From 5e7d13df89b45932cdc054fd1123d57c49f12dc1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 8 Nov 2019 10:59:34 +0100 Subject: [PATCH 21/26] Update uppmax.md --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index d5bba37..61a1832 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -26,7 +26,7 @@ If running on Bianca, you will have no internet connection and these configs wil Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. -Please note that Bianca only allocates 7 GB memory per core so max memory needs to be configured: +Note that Bianca only allocates 7 GB memory per core so the max memory needs to be limited: ```bash --max_memory "112GB" From d2603a134ed9e1922ad18db8fe747140ae8d8391 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 12:26:19 +0100 Subject: [PATCH 22/26] Add in denbi cloud configs --- conf/denbi_qbic.config | 26 ++++++++++++++++++++++++++ docs/denbi_qbic.md | 8 ++++++++ 2 files changed, 34 insertions(+) create mode 100644 conf/denbi_qbic.config create mode 100644 docs/denbi_qbic.md diff --git a/conf/denbi_qbic.config b/conf/denbi_qbic.config new file mode 100644 index 0000000..d5f1882 --- /dev/null +++ b/conf/denbi_qbic.config @@ -0,0 +1,26 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'de.NBI cluster profile provided by nf-core/configs.' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://cloud.denbi.de/' +} + +singularity { + enabled = true +} + +process { + executor = 'pbs' + queue = 'batch' +} + +params { + max_memory = 512.GB + max_cpus = 28 + max_time = 960.h +} + +weblog{ + enabled = true + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' +} diff --git a/docs/denbi_qbic.md b/docs/denbi_qbic.md new file mode 100644 index 0000000..220aac2 --- /dev/null +++ b/docs/denbi_qbic.md @@ -0,0 +1,8 @@ +# nf-core/configs: de.NBI QBIC Configuration + +All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). + +To use, run the pipeline with `-profile denbi_qbic`. This will download and launch the [`denbi_qbic.config`](../conf/denbi_qbic.config) which has been pre-configured with a setup suitable for the automatically created cluster. Using this profile, a Docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +>NB: You will need an account to use de.NBI Cluster in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 3b6c9d0360573b4475dffb0ccd97bc1fc157e624 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 12:28:45 +0100 Subject: [PATCH 23/26] Fix denbi docs --- README.md | 1 + nfcore_custom.config | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 20fafd7..d9cc35f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Currently documentation is available for the following clusters: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [DENBI_QBIC](docs/denbi_qbic.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index 12a0c1e..9ec9ade 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } From bc9ecf50acc8f8ca1b8a9e92229f0666223ec2f5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 13:56:17 +0100 Subject: [PATCH 24/26] Update binac.config Upgrade to 3.4.2 --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index d3c9a3c..e34f615 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -10,7 +10,7 @@ singularity { } process { - beforeScript = 'module load devel/singularity/3.0.3' + beforeScript = 'module load devel/singularity/3.4.2' executor = 'pbs' queue = 'short' process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'short' : 'long'} From 08a6b748250c26147fc61b827598fef6a5e0a22a Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 10 Oct 2019 15:33:06 +0200 Subject: [PATCH 25/26] add new profile to be shared: awsbatch --- README.md | 9 +++++---- conf/awsbatch.config | 14 ++++++++++++++ docs/awsbatch.md | 4 ++++ nfcore_custom.config | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 conf/awsbatch.config create mode 100644 docs/awsbatch.md diff --git a/README.md b/README.md index d9cc35f..3ef16e0 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,15 @@ A repository for hosting nextflow config files containing custom parameters requ ## Using an existing config -The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. +The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organization are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organization. ### Configuration and parameters The config files hosted in this repository define a set of parameters which are specific to compute environments at different Institutions but generic enough to be used with all nf-core pipelines. -All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customise the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. +All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. -You should be able to get a good idea as to how other people are customising the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). +You should be able to get a good idea as to how other people are customizing the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). ### Offline usage @@ -80,8 +80,9 @@ You will have to create a [Markdown document](https://www.markdownguide.org/gett See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs) for examples. -Currently documentation is available for the following clusters: +Currently documentation is available for the following systems: +* [AWSBATCH](docs/awsbatch.md) * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) diff --git a/conf/awsbatch.config b/conf/awsbatch.config new file mode 100644 index 0000000..978da56 --- /dev/null +++ b/conf/awsbatch.config @@ -0,0 +1,14 @@ +//Nextflow config file for running on AWS batch + +params { + config_profile_name = 'AWSBATCH' + config_profile_description = 'AWSBATCH Cloud Profile' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://aws.amazon.com/batch/' +} + +aws.region = params.awsregion +process.executor = 'awsbatch' +process.queue = params.awsqueue +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' \ No newline at end of file diff --git a/docs/awsbatch.md b/docs/awsbatch.md new file mode 100644 index 0000000..bec4d78 --- /dev/null +++ b/docs/awsbatch.md @@ -0,0 +1,4 @@ +# nf-core/configs: awsbatch Configuration + +To be used with `awsbatch`. +Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 9ec9ade..cceb01c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } From cb1c2eacd05476a6a359e1602a95b9bbc91bf92f Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 19 Nov 2019 16:36:51 +0100 Subject: [PATCH 26/26] Z->S --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ef16e0..b9812b6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A repository for hosting nextflow config files containing custom parameters requ ## Using an existing config -The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organization are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organization. +The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. ### Configuration and parameters @@ -24,7 +24,7 @@ The config files hosted in this repository define a set of parameters which are All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. -You should be able to get a good idea as to how other people are customizing the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). +You should be able to get a good idea as to how other people are customising the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). ### Offline usage