From ac4372b610a33a0b3e7daeccd417c2342482d1c5 Mon Sep 17 00:00:00 2001 From: Brad Langhorst Date: Tue, 3 Aug 2021 13:33:31 -0400 Subject: [PATCH 01/94] adds varskip --- conf/pipeline/viralrecon/genomes.config | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 3e3e299..25bce38 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -50,6 +50,20 @@ params { scheme = 'nCoV-2019' } } + 'NEB' { + 'vss1' { + fasta = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/primer_schemes/artic/nCoV-2019/V1200/nCoV-2019.reference.fasta' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/primer_schemes/NEB/nCoV-2019/vss1/neb_vss1.primer.bed' + scheme = 'nCoV-2019' + } + 'vsl1' { + fasta = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/primer_schemes/artic/nCoV-2019/V1200/nCoV-2019.reference.fasta' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/primer_schemes/NEB/nCoV-2019/vsl1/neb_vsl1.primer.bed' + scheme = 'nCoV-2019' + } + } } } } From 94d5a1b5f27b535851fbf32d6a6b09d5a1fdc214 Mon Sep 17 00:00:00 2001 From: Kevin Brick Date: Thu, 23 Dec 2021 15:36:38 -0500 Subject: [PATCH 02/94] Update nihbiowulf.config --- conf/nihbiowulf.config | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conf/nihbiowulf.config b/conf/nihbiowulf.config index 6159f0d..92deee0 100644 --- a/conf/nihbiowulf.config +++ b/conf/nihbiowulf.config @@ -28,6 +28,15 @@ profiles { executor.$slurm.submitRateLimit = '6/1min' process.clusterOptions = ' --gres=lscratch:600 --signal USR2@20' } + + standard { + process.executor = 'slurm' + executor.$slurm.pollInterval = '1 min' + executor.$slurm.queueStatInterval = '5 min' + executor.queueSize = 100 + executor.$slurm.submitRateLimit = '6/1min' + process.clusterOptions = ' --gres=lscratch:600 --signal USR2@20' + } } singularity { From 06854c9f2d7fda372a932347485524f116e055cd Mon Sep 17 00:00:00 2001 From: ajodeh-juma Date: Wed, 16 Mar 2022 15:34:36 +0300 Subject: [PATCH 03/94] add gis.md documentation --- docs/gis.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docs/gis.md diff --git a/docs/gis.md b/docs/gis.md new file mode 100644 index 0000000..0217781 --- /dev/null +++ b/docs/gis.md @@ -0,0 +1,45 @@ +# nf-core/configs: GIS Aquila Configuration + +All nf-core pipelines have been successfully configured for use on the cluster of the GIS (Genome Institute of Singapore (Aquila)). + +To use, run the pipeline with `-profile gis`. This will download and launch the [`gis.config`](../conf/gis.config) which has been pre-configured with a setup suitable for the GIS Aquila cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## How to use on GIS core + +Before running the pipeline you will need to load Nextflow using the environment module system on GIS Aquila. You can do this by issuing the commands below: + +```bash +# Login to a compute node +srun --pty bash + +## Load Nextflow and Singularity environment modules +module purge +source /mnt/projects/rpd/rc/init.2017-04 +module load miniconda3 + + +# Run a nextflow pipeline with dependencies bundles in a conda environment +set +u +source activate nfcore-rnaseq-1.0dev +set -u + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +nf-core/workflow \\ +-resume +-profile gis \\ +--email my-email@example.org \\ +-c my-specific.config +... + + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... + +``` + +## Databanks + +A local copy of several genomes are available in `/mnt/projects/rpd/genomes.testing/S3_igenomes/` directory. + +>NB: You will need an account to use the HPC cluster on GIS in order to run the pipeline. If in doubt contact IT or go to [Andreas Wilm](https://github.com/andreas-wilm) From 188c3a39cc11ab7251b42d8d7b0b65785e67a126 Mon Sep 17 00:00:00 2001 From: ajodeh-juma Date: Wed, 16 Mar 2022 16:17:54 +0300 Subject: [PATCH 04/94] add gis.md documentation --- docs/gis.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gis.md b/docs/gis.md index 0217781..877807d 100644 --- a/docs/gis.md +++ b/docs/gis.md @@ -26,7 +26,7 @@ set -u # Run a downloaded/git-cloned nextflow workflow from nextflow run \\ nf-core/workflow \\ --resume +-resume \\ -profile gis \\ --email my-email@example.org \\ -c my-specific.config @@ -40,6 +40,6 @@ nextflow run nf-core/rnaseq ... ## Databanks -A local copy of several genomes are available in `/mnt/projects/rpd/genomes.testing/S3_igenomes/` directory. +A local copy of several genomes are available in `/mnt/projects/rpd/genomes.testing/S3_igenomes/` directory. >NB: You will need an account to use the HPC cluster on GIS in order to run the pipeline. If in doubt contact IT or go to [Andreas Wilm](https://github.com/andreas-wilm) From 28c3cca5e7cc55a1920fe7c834a52491cdc2aa63 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 7 Jun 2022 10:03:04 +0000 Subject: [PATCH 05/94] [automated] Fix linting with Prettier --- docs/gis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gis.md b/docs/gis.md index 877807d..1864215 100644 --- a/docs/gis.md +++ b/docs/gis.md @@ -42,4 +42,4 @@ nextflow run nf-core/rnaseq ... A local copy of several genomes are available in `/mnt/projects/rpd/genomes.testing/S3_igenomes/` directory. ->NB: You will need an account to use the HPC cluster on GIS in order to run the pipeline. If in doubt contact IT or go to [Andreas Wilm](https://github.com/andreas-wilm) +> NB: You will need an account to use the HPC cluster on GIS in order to run the pipeline. If in doubt contact IT or go to [Andreas Wilm](https://github.com/andreas-wilm) From ba8bffbdfdb69fb031bcef52dc64985c680716f2 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 15 Jun 2022 15:39:36 +0200 Subject: [PATCH 06/94] Use env scope instead of beforescript for mag@eva --- conf/pipeline/mag/eva.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config index 81d8d0c..37a531d 100644 --- a/conf/pipeline/mag/eva.config +++ b/conf/pipeline/mag/eva.config @@ -4,6 +4,11 @@ params { config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' } +env { + OPENBLAS_NUM_THREADS=1 + OMP_NUM_THREADS=1 +} + process { withName: FASTQC { From 51d057564f9fc43248f718a2aff0f710ff3ee8e3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 15 Jun 2022 15:40:42 +0200 Subject: [PATCH 07/94] Fix python related seg faults mag@eva --- conf/pipeline/mag/eva.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config index 81d8d0c..37a531d 100644 --- a/conf/pipeline/mag/eva.config +++ b/conf/pipeline/mag/eva.config @@ -4,6 +4,11 @@ params { config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' } +env { + OPENBLAS_NUM_THREADS=1 + OMP_NUM_THREADS=1 +} + process { withName: FASTQC { From 65d7080921eb37bd7544df601dc4cb3206102131 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Tue, 5 Jul 2022 11:17:49 +0200 Subject: [PATCH 08/94] add conf/ku_sund_dangpu.config --- conf/ku_sund_dangpu.config | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 conf/ku_sund_dangpu.config diff --git a/conf/ku_sund_dangpu.config b/conf/ku_sund_dangpu.config new file mode 100644 index 0000000..3f3ab5d --- /dev/null +++ b/conf/ku_sund_dangpu.config @@ -0,0 +1,36 @@ +# change ${abc123} to your KU ID before using this config + +params { + config_profile_contact = ‘Adrija Kalvisa ’ + config_profile_description = ‘dangpufl01 configuration’ + config_profile_url = ‘’ + + // General cpus/memory/time requirements + max_cpus = 30 + max_memory = 200.GB + max_time = 72.h +} + +process { + # should we change to a newer java env? + beforeScript = ‘module load jdk/1.8.0_291 singularity/3.8.0’ + executor = ‘slurm’ + + # uncomment specifically for running mnaseseq pipeline: + # withName: TrimGalore { + # cpus = 1 + # } + +} + +executor { + queueSize = 10 +} + +singularity { + enabled = true + autoMounts = true + # set up the cache directory outside the $HOME directory + cacheDir = ‘/projects/dan1/people/${abc123}/cache/singularity-images’ + runOptions = ‘--bind /projects:/projects’ +} \ No newline at end of file From 8ba35f721c496b081192cd38f3b2c2e4936f259e Mon Sep 17 00:00:00 2001 From: qlr900 Date: Tue, 5 Jul 2022 12:05:10 +0200 Subject: [PATCH 09/94] add ku_sund_dangpu.md --- docs/ku_sund_dangpu.md | 58 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 docs/ku_sund_dangpu.md diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md new file mode 100644 index 0000000..c2b9454 --- /dev/null +++ b/docs/ku_sund_dangpu.md @@ -0,0 +1,58 @@ +# nf-core/configs: PROFILE Configuration + +All nf-core pipelines have been successfully configured for use on the dangpu PROFILE CLUSTER at the +Novo Nordisk Foundation Center for Stem Cell Medicine (reNEW) and the Novo Nordisk Foundation Center for Protein Research (CPR) at the University of Copenhagen. + +To use, run the pipeline with `-profile ku_sund_dangpu`. This will download and launch the [`ku_sund_dangpu.config`](../conf/ku_sund_dangpu.config) which has been pre-configured with a setup suitable for the PROFILE CLUSTER. + +## Modules + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on PROFILE CLUSTER. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 +``` + +## How to use the nf-core pipelines on PROFILE CLUSTER + +###1. set up .bash_profile +Add memory restrictions to `.bash_profile`. Use `vim` text editor to edit `.bash_profile` file (or create one if it does not exist in your home directory: + +``` +vi ~/.bash_profile +``` + +paste the following text within the file: +``` +# personalising the profile: change the value of the variable $abc123 to your own user id +abc123=def456 + +# In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. We recommend adding the following line to your environment to limit this +export NXF_OPTS='-Xms1g -Xmx4g' + +# Don't fill up your home directory with cache files +export NXF_HOME=/projects/dan1/people/${abc123}/cache/nxf-home +export NXF_TEMP=${SNIC_TMP:-$HOME/glob/nxftmp} + +# Nextflow singularity image cachedir export +NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${abc123}/cache/singularity-images +``` + +then save the changes and exit the editor. Now you need to activate the `.bash_profile` by logging out and in again to PROFILE CLUSTER or by typing: +``` +source .bash_profile +``` + +The `$HOME` directory at dangpu has restricted space and should not be used to store cache files. Create those designated cache spaces for nextflow and singularity outside your `$HOME`: +``` +mkdir $NXF_SINGULARITY_CACHEDIR +mkdir $NXF_HOME +``` + +### 3. choose a nf-core pipeline and test it with your preferred settings +``` +nextflow run nf-core/rnaseq -profile test -c +``` + From 941a34406aac86bf92c5523a48cea120590ac4e7 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Tue, 5 Jul 2022 12:06:19 +0200 Subject: [PATCH 10/94] update ku_sund_dangpu.md --- docs/ku_sund_dangpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index c2b9454..9dafaa0 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -53,6 +53,6 @@ mkdir $NXF_HOME ### 3. choose a nf-core pipeline and test it with your preferred settings ``` -nextflow run nf-core/rnaseq -profile test -c +nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu ``` From 7d59eabd91635879edab05ef9ec8f007b37e5bf4 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 09:52:06 +0200 Subject: [PATCH 11/94] simplify guideline --- docs/ku_sund_dangpu.md | 43 ++++++++---------------------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 9dafaa0..6121c87 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -1,58 +1,31 @@ # nf-core/configs: PROFILE Configuration -All nf-core pipelines have been successfully configured for use on the dangpu PROFILE CLUSTER at the +All nf-core pipelines have been successfully configured for use on the DANGPU at the Novo Nordisk Foundation Center for Stem Cell Medicine (reNEW) and the Novo Nordisk Foundation Center for Protein Research (CPR) at the University of Copenhagen. -To use, run the pipeline with `-profile ku_sund_dangpu`. This will download and launch the [`ku_sund_dangpu.config`](../conf/ku_sund_dangpu.config) which has been pre-configured with a setup suitable for the PROFILE CLUSTER. +To use, run the pipeline with `-profile ku_sund_dangpu`. This will download and launch the [`ku_sund_dangpu.config`](../conf/ku_sund_dangpu.config) which has been pre-configured with a setup suitable for the DANGPU. ## Modules -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on PROFILE CLUSTER. You can do this by issuing the commands below: +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on DANGPU. You can do this by issuing the commands below: ```bash ## Load Nextflow and Singularity environment modules module purge module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 -``` - -## How to use the nf-core pipelines on PROFILE CLUSTER - -###1. set up .bash_profile -Add memory restrictions to `.bash_profile`. Use `vim` text editor to edit `.bash_profile` file (or create one if it does not exist in your home directory: - -``` -vi ~/.bash_profile -``` - -paste the following text within the file: -``` -# personalising the profile: change the value of the variable $abc123 to your own user id -abc123=def456 - -# In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. We recommend adding the following line to your environment to limit this export NXF_OPTS='-Xms1g -Xmx4g' - -# Don't fill up your home directory with cache files -export NXF_HOME=/projects/dan1/people/${abc123}/cache/nxf-home -export NXF_TEMP=${SNIC_TMP:-$HOME/glob/nxftmp} - -# Nextflow singularity image cachedir export -NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${abc123}/cache/singularity-images +export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home +export NXF_TEMP=/scratch/tmp +NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images ``` -then save the changes and exit the editor. Now you need to activate the `.bash_profile` by logging out and in again to PROFILE CLUSTER or by typing: -``` -source .bash_profile -``` - -The `$HOME` directory at dangpu has restricted space and should not be used to store cache files. Create those designated cache spaces for nextflow and singularity outside your `$HOME`: +create nextflow directories if they dont exist: ``` mkdir $NXF_SINGULARITY_CACHEDIR mkdir $NXF_HOME ``` -### 3. choose a nf-core pipeline and test it with your preferred settings +finally download and test the pipeline of choice using the `-profile ku_sund_dangpu` ``` nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu ``` - From 56026c5ac4fec88f85eb1e0a44b2ea4e3d053853 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 09:52:51 +0200 Subject: [PATCH 12/94] remove beforeScript --- conf/ku_sund_dangpu.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/ku_sund_dangpu.config b/conf/ku_sund_dangpu.config index 3f3ab5d..442b8af 100644 --- a/conf/ku_sund_dangpu.config +++ b/conf/ku_sund_dangpu.config @@ -12,8 +12,7 @@ params { } process { - # should we change to a newer java env? - beforeScript = ‘module load jdk/1.8.0_291 singularity/3.8.0’ + executor = ‘slurm’ # uncomment specifically for running mnaseseq pipeline: From 49e756d1bc94d3070c08232233ffc374bcdf5536 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 12:39:55 +0200 Subject: [PATCH 13/94] fix typos --- conf/ku_sund_dangpu.config | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/conf/ku_sund_dangpu.config b/conf/ku_sund_dangpu.config index 442b8af..7712376 100644 --- a/conf/ku_sund_dangpu.config +++ b/conf/ku_sund_dangpu.config @@ -1,9 +1,7 @@ -# change ${abc123} to your KU ID before using this config - params { - config_profile_contact = ‘Adrija Kalvisa ’ - config_profile_description = ‘dangpufl01 configuration’ - config_profile_url = ‘’ + config_profile_contact = 'Adrija Kalvisa ' + config_profile_description = 'dangpufl01 configuration' + config_profile_url = '' // General cpus/memory/time requirements max_cpus = 30 @@ -13,7 +11,7 @@ params { process { - executor = ‘slurm’ + executor = 'slurm' # uncomment specifically for running mnaseseq pipeline: # withName: TrimGalore { @@ -30,6 +28,6 @@ singularity { enabled = true autoMounts = true # set up the cache directory outside the $HOME directory - cacheDir = ‘/projects/dan1/people/${abc123}/cache/singularity-images’ - runOptions = ‘--bind /projects:/projects’ + cacheDir = '/projects/dan1/people/$USER/cache/singularity-images' + runOptions = '--bind /projects:/projects' } \ No newline at end of file From 7ac08a101577c04283b7592d6583d6ab17551d42 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 12:42:14 +0200 Subject: [PATCH 14/94] add alternative module load --- docs/ku_sund_dangpu.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 6121c87..92e73b5 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -13,6 +13,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 +# alternative module load jdk/1.8.0_291 nextflow/22.04.3 singularity/3.8.0 export NXF_OPTS='-Xms1g -Xmx4g' export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home export NXF_TEMP=/scratch/tmp From 8a781b6bb2267cca0b1d4ec87da2a7c577e69906 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 12:54:07 +0200 Subject: [PATCH 15/94] modify alternative module load --- docs/ku_sund_dangpu.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 92e73b5..f63aa0c 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -13,7 +13,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 -# alternative module load jdk/1.8.0_291 nextflow/22.04.3 singularity/3.8.0 +# alternative module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 export NXF_OPTS='-Xms1g -Xmx4g' export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home export NXF_TEMP=/scratch/tmp @@ -26,7 +26,9 @@ mkdir $NXF_SINGULARITY_CACHEDIR mkdir $NXF_HOME ``` -finally download and test the pipeline of choice using the `-profile ku_sund_dangpu` +Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm as resource manager within the ku_sund_dangpu profile. Just make sure that the pipeline is run within a tmux session. + ``` nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu ``` + From f45b8645f63a5c25a16d66626546bc1d96ea86b5 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 13:05:13 +0200 Subject: [PATCH 16/94] update wording --- docs/ku_sund_dangpu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index f63aa0c..5515b87 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -26,7 +26,7 @@ mkdir $NXF_SINGULARITY_CACHEDIR mkdir $NXF_HOME ``` -Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm as resource manager within the ku_sund_dangpu profile. Just make sure that the pipeline is run within a tmux session. +Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm as resource manager within the `ku_sund_dangpu profile`. Just make sure that the pipeline is run within a tmux session. ``` nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu From 6939c6f0033e0bc419ff44747d0fd9a2fa1dae99 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 13:34:18 +0200 Subject: [PATCH 17/94] lower queueSize to 5 --- conf/ku_sund_dangpu.config | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conf/ku_sund_dangpu.config b/conf/ku_sund_dangpu.config index 7712376..bb70515 100644 --- a/conf/ku_sund_dangpu.config +++ b/conf/ku_sund_dangpu.config @@ -21,13 +21,11 @@ process { } executor { - queueSize = 10 + queueSize = 5 } singularity { enabled = true autoMounts = true - # set up the cache directory outside the $HOME directory - cacheDir = '/projects/dan1/people/$USER/cache/singularity-images' runOptions = '--bind /projects:/projects' } \ No newline at end of file From 4f98876429cb5ba81ed7f4564670a90d0e4d40b7 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 13:35:10 +0200 Subject: [PATCH 18/94] add export to NXF_SINGULARITY_CACHEDIR --- docs/ku_sund_dangpu.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 5515b87..96bff9f 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -1,4 +1,4 @@ -# nf-core/configs: PROFILE Configuration +# nf-core/configs: ku_sund_dangpu configuration All nf-core pipelines have been successfully configured for use on the DANGPU at the Novo Nordisk Foundation Center for Stem Cell Medicine (reNEW) and the Novo Nordisk Foundation Center for Protein Research (CPR) at the University of Copenhagen. @@ -13,11 +13,12 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 -# alternative module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 +# alternative +# module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 export NXF_OPTS='-Xms1g -Xmx4g' export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home export NXF_TEMP=/scratch/tmp -NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images +export NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images ``` create nextflow directories if they dont exist: From d2b6b07de81be19c081fd610c20eccaf6389abce Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 14:18:31 +0200 Subject: [PATCH 19/94] remove comments for mnaseseq --- conf/ku_sund_dangpu.config | 6 ------ 1 file changed, 6 deletions(-) diff --git a/conf/ku_sund_dangpu.config b/conf/ku_sund_dangpu.config index bb70515..51ca462 100644 --- a/conf/ku_sund_dangpu.config +++ b/conf/ku_sund_dangpu.config @@ -10,14 +10,8 @@ params { } process { - executor = 'slurm' - # uncomment specifically for running mnaseseq pipeline: - # withName: TrimGalore { - # cpus = 1 - # } - } executor { From d19d6c0ae0e545202ad8983accf2c83e1e040ca1 Mon Sep 17 00:00:00 2001 From: qlr900 Date: Wed, 6 Jul 2022 16:19:16 +0200 Subject: [PATCH 20/94] update wording --- docs/ku_sund_dangpu.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 96bff9f..5c6fc4e 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -21,13 +21,13 @@ export NXF_TEMP=/scratch/tmp export NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images ``` -create nextflow directories if they dont exist: +Create the user-specific nextflow directories if they don't exist yet: ``` mkdir $NXF_SINGULARITY_CACHEDIR mkdir $NXF_HOME ``` -Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm as resource manager within the `ku_sund_dangpu profile`. Just make sure that the pipeline is run within a tmux session. +Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm to be the resource manager within the `ku_sund_dangpu profile`. Just make sure that the pipeline is run within a tmux session. ``` nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu From ff55ec67b4672376e3754b898f21be2c2d112cbd Mon Sep 17 00:00:00 2001 From: qlr900 Date: Thu, 7 Jul 2022 14:30:48 +0200 Subject: [PATCH 21/94] update to current java+nextflow module combination --- docs/ku_sund_dangpu.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index 5c6fc4e..aea1b40 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -12,8 +12,8 @@ Before running the pipeline you will need to load Nextflow and Singularity using ```bash ## Load Nextflow and Singularity environment modules module purge -module load openjdk/11.0.0 nextflow/22.04.3 singularity/3.8.0 -# alternative +module load java/11.0.15 nextflow/22.04.4 singularity/3.8.0 +# alternative modules for older nextflow version (v.21) that works with java 8: # module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 export NXF_OPTS='-Xms1g -Xmx4g' export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home From 8448bb9a1edb7aaf4103077f4d36e3a971e5ad7c Mon Sep 17 00:00:00 2001 From: Lara Ianov <61988230+lianov@users.noreply.github.com> Date: Thu, 1 Sep 2022 10:12:07 -0500 Subject: [PATCH 22/94] re-define TMPDIR --- conf/cheaha.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cheaha.config b/conf/cheaha.config index 58963b5..ec79f10 100644 --- a/conf/cheaha.config +++ b/conf/cheaha.config @@ -9,7 +9,7 @@ params { } env { - TMPDIR="$USER" + TMPDIR="$scratch_dir" SINGULARITY_TMPDIR="$scratch_dir" } From b20f28a3c9f3e4df21030b179a46ddb2fe151307 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 5 Sep 2022 11:25:33 +0200 Subject: [PATCH 23/94] biohpc_gen: load charliecloud from lrz software tree and remove conda fallback the biohpc_gen specific software tree is less maintained and outdated --- conf/biohpc_gen.config | 9 ++++----- docs/biohpc_gen.md | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/conf/biohpc_gen.config b/conf/biohpc_gen.config index e3f4069..694a25f 100755 --- a/conf/biohpc_gen.config +++ b/conf/biohpc_gen.config @@ -11,9 +11,8 @@ env { process { executor = 'slurm' - queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } - beforeScript = 'module use /dss/dsslegfs02/pn73se/pn73se-dss-0000/spack/modules/x86_avx2/linux*' - module = 'charliecloud/0.22:miniconda3' + queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } + module = 'charliecloud/0.25' } charliecloud { @@ -21,7 +20,7 @@ charliecloud { } params { - params.max_time = 14.d - params.max_cpus = 80 + params.max_time = 14.d + params.max_cpus = 80 params.max_memory = 3.TB } diff --git a/docs/biohpc_gen.md b/docs/biohpc_gen.md index c007cad..660e789 100644 --- a/docs/biohpc_gen.md +++ b/docs/biohpc_gen.md @@ -4,14 +4,12 @@ All nf-core pipelines have been successfully configured for use on the BioHPC Ge To use, run the pipeline with `-profile biohpc_gen`. This will download and launch the [`biohpc_gen.config`](../conf/biohpc_gen.config) which has been pre-configured with a setup suitable for the biohpc_gen cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Charliecloud container before execution of the pipeline. -Before running the pipeline you will need to load Nextflow and Charliecloud using the environment module system on biohpc_gen. You can do this by issuing the commands below: +Before running the pipeline you will need to load Nextflow and Charliecloud using the environment module system on a login node. You can do this by issuing the commands below: ```bash ## Load Nextflow and Charliecloud environment modules -module purge -module load nextflow charliecloud/0.22 +module load nextflow/21.04.3 charliecloud/0.25 ``` -> NB: Charliecloud support requires Nextflow version `21.03.0-edge` or later. > NB: You will need an account to use the LRZ Linux cluster as well as group access to the biohpc_gen cluster in order to run nf-core pipelines. > NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. From abae5c4bd897b79da97a6fb2ff6c990d6bd04c0b Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Mon, 12 Sep 2022 11:40:43 -0700 Subject: [PATCH 24/94] Decrease AWS Batch queue size --- conf/sage.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/sage.config b/conf/sage.config index bfe1e09..d43e719 100644 --- a/conf/sage.config +++ b/conf/sage.config @@ -34,7 +34,7 @@ aws { executor { name = 'awsbatch' // Ensure unlimited queue size on AWS Batch - queueSize = 100000 + queueSize = 500 // Slow down the rate at which AWS Batch jobs accumulate in // the queue (an attempt to prevent orphaned EBS volumes) submitRateLimit = '5 / 1 sec' From a9b62c53e0bece4fb578ff1168f72e820c3be6b9 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 13 Sep 2022 17:02:44 +0100 Subject: [PATCH 25/94] Added global institutional and pipeline configs for sbc_sharc --- .github/workflows/main.yml | 1 + README.md | 1 + conf/pipeline/atacseq/sbc_sharc.config | 74 ++++++ conf/pipeline/chipseq/sbc_sharc.config | 74 ++++++ conf/pipeline/rnaseq/sbc_sharc.config | 79 ++++++ conf/pipeline/sarek/sbc_sharc.config | 114 ++++++++ conf/sbc_sharc.config | 59 ++++ docs/sbc_sharc.md | 355 +++++++++++++++++++++++++ nfcore_custom.config | 1 + pipeline/atacseq.config | 13 + pipeline/chipseq.config | 13 + pipeline/rnaseq.config | 1 + pipeline/sarek.config | 1 + 13 files changed, 786 insertions(+) create mode 100644 conf/pipeline/atacseq/sbc_sharc.config create mode 100644 conf/pipeline/chipseq/sbc_sharc.config create mode 100644 conf/pipeline/rnaseq/sbc_sharc.config create mode 100644 conf/pipeline/sarek/sbc_sharc.config create mode 100644 conf/sbc_sharc.config create mode 100644 docs/sbc_sharc.md create mode 100644 pipeline/atacseq.config create mode 100644 pipeline/chipseq.config diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 487c7ee..ca01a15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -83,6 +83,7 @@ jobs: - "sage" - "sahmri" - "sanger" + - "sbc_sharc" - "seg_globe" - "uct_hpc" - "unibe_ibu" diff --git a/README.md b/README.md index 445467b..0cad49f 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ Currently documentation is available for the following systems: - [ROSALIND](docs/rosalind.md) - [SAGE BIONETWORKS](docs/sage.md) - [SANGER](docs/sanger.md) +- [SBC_SHARC](docs/sbc_sharc.md) - [SEG_GLOBE](docs/seg_globe.md) - [UCT_HPC](docs/uct_hpc.md) - [UNIBE_IBU](docs/unibe_ibu.md) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config new file mode 100644 index 0000000..2e987d2 --- /dev/null +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -0,0 +1,74 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Sheffield Bioinformatics Core Configuration Profile - ShARC + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Custom Pipeline Resource Config for nf-core/atacseq + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + +// process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config + +process { + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_high { + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_long { + time = { check_max( 12.h * task.attempt, 'time' ) } + } + +} + + +// function 'check_max()' to ensure that resource requirements don't go beyond maximum limit + +def check_max(obj, type) { + if (type == 'memory') { + try { + if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) + return params.max_memory as nextflow.util.MemoryUnit + else + return obj + } catch (all) { + println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'time') { + try { + if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) + return params.max_time as nextflow.util.Duration + else + return obj + } catch (all) { + println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'cpus') { + try { + return Math.min(obj, params.max_cpus as int) + } catch (all) { + println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" + return obj + } + } +} diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config new file mode 100644 index 0000000..2741453 --- /dev/null +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -0,0 +1,74 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Sheffield Bioinformatics Core Configuration Profile - ShARC + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Custom Pipeline Resource Config for nf-core/chipseq + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + +// process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config + +process { + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_high { + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_long { + time = { check_max( 12.h * task.attempt, 'time' ) } + } + +} + + +// function 'check_max()' to ensure that resource requirements don't go beyond maximum limit + +def check_max(obj, type) { + if (type == 'memory') { + try { + if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) + return params.max_memory as nextflow.util.MemoryUnit + else + return obj + } catch (all) { + println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'time') { + try { + if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) + return params.max_time as nextflow.util.Duration + else + return obj + } catch (all) { + println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'cpus') { + try { + return Math.min(obj, params.max_cpus as int) + } catch (all) { + println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" + return obj + } + } +} diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config new file mode 100644 index 0000000..52bf0ff --- /dev/null +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -0,0 +1,79 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Sheffield Bioinformatics Core Configuration Profile - ShARC + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Custom Pipeline Resource Config for nf-core/rnaseq + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + +// process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config + +process { + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_high { + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_long { + time = { check_max( 12.h * task.attempt, 'time' ) } + } + + withLabel:process_high_memory { + memory = { check_max( 60.GB * task.attempt, 'memory' ) } + } + +} + + +// function 'check_max()' to ensure that resource requirements don't go beyond maximum limit + +def check_max(obj, type) { + if (type == 'memory') { + try { + if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) + return params.max_memory as nextflow.util.MemoryUnit + else + return obj + } catch (all) { + println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'time') { + try { + if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) + return params.max_time as nextflow.util.Duration + else + return obj + } catch (all) { + println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'cpus') { + try { + return Math.min(obj, params.max_cpus as int) + } catch (all) { + println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" + return obj + } + } +} + diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config new file mode 100644 index 0000000..204d73b --- /dev/null +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -0,0 +1,114 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Sheffield Bioinformatics Core Configuration Profile - ShARC + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Custom Pipeline Resource Config for nf-core/sarek + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + +// process-specific resource requirements - reduced specification from those in sarek/conf/base.config + +process { + + + // process labels + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_high { + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_long { + time = { check_max( 12.h * task.attempt, 'time' ) } + } + + withLabel:process_high_memory { + memory = { check_max( 60.GB * task.attempt, 'memory' ) } + } + + + // process name + + withName:'BWAMEM1_MEM|BWAMEM2_MEM' { + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withName:'FASTP' { + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + } + + withName:'FASTQC|FASTP|MOSDEPTH|SAMTOOLS_CONVERT' { + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + } + + withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|SAMTOOLS_STATS' { + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + } + + withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + } + + withName:'GATK4_MARKDUPLICATES' { + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + } + + withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { + cpus = { check_max( 1 * task.attempt, 'cpus' ) } + } + +} + + +// function 'check_max()' to ensure that resource requirements don't go beyond maximum limit + +def check_max(obj, type) { + if (type == 'memory') { + try { + if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) + return params.max_memory as nextflow.util.MemoryUnit + else + return obj + } catch (all) { + println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'time') { + try { + if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) + return params.max_time as nextflow.util.Duration + else + return obj + } catch (all) { + println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'cpus') { + try { + return Math.min(obj, params.max_cpus as int) + } catch (all) { + println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" + return obj + } + } +} diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config new file mode 100644 index 0000000..ca41185 --- /dev/null +++ b/conf/sbc_sharc.config @@ -0,0 +1,59 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Sheffield Bioinformatics Core Configuration Profile - ShARC + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Base Institutional Configuration + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + + +params { + + // nf-core specific parameters displayed in header summary of each run + + config_profile_description = 'Sheffield Bioinformatics Core - ShARC' + config_profile_contact = 'Lewis Quayle (l.quayle@sheffield.ac.uk)' + config_profile_url = 'https://docs.hpc.shef.ac.uk/en/latest/sharc/index.html' + + // hpc resource limits + + max_cpus = 16 + max_memory = 64.GB + max_time = 96.h + +} + + +// container engine + +singularity { + + enabled = true + autoMounts = true + // cacheDir = '////' + +} + + +// hpc configuration specific to ShARC + +process { + + // scheduler + + executor = 'sge' + penv = 'smp' + queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } + clusterOptions = { "-l rmem=${task.memory.toGiga()}G" } + + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + +} + diff --git a/docs/sbc_sharc.md b/docs/sbc_sharc.md new file mode 100644 index 0000000..7b055d1 --- /dev/null +++ b/docs/sbc_sharc.md @@ -0,0 +1,355 @@ +# nf-core/configs: Sheffield Bioinformatics Core Facility ShARC Configuration + +The following nf-core pipelines have been successfully configured for use on the the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html): + +- [atacseq](https://nf-co.re/atacseq) +- [chipseq](https://nf-co.re/chipseq) +- [rnaseq](https://nf-co.re/rnaseq) +- [sarek](https://nf-co.re/sarek) + +When using [`sbc_sharc.config`](../conf/sbc_sharc.config) with the pipelines listed above, the appropriate configuration file from the list below will be loaded automatically: + +- atacseq: [sbc_sharc atacseq config](../conf/pipeline/atacseq/sbc_sharc.config) +- chipseq: [sbc_sharc chipseq config](../conf/pipeline/chipseq/sbc_sharc.config) +- rnaseq: [sbc_sharc rnaseq config](../conf/pipeline/rnaseq/sbc_sharc.config) +- sarek: [sbc_sharc sarek config](../conf/pipeline/sarek/sbc_sharc.config) + +The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file might work with other nf-core pipelines as it stands but we cannot guarantee they will run without issue. We will be continuing to create, test and optimise configurations for new pipelines in the future. + + +## Using the SBC_ShARC Institutional Configuration Profile + +To use [`sbc_sharc.config`](../conf/sbc_sharc.config), run nextflow with an nf-core pipeline using `-profile sbc_sharc` (note the single hyphen). This will download and launch [`sbc_sharc.config`](../conf/sbc_sharc.config) which has been pre-configured with a setup suitable for the ShARC cluster and will automatically load the appropriate pipeline-specific configuration file. + +For a full guide on how to setup and run Nextflow using nf-core pipelines on ShARC, see the **Running Nextflow with nf-core Pipelines on ShARC** section below. + + +## A Note on Singularity Containers + +The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file supports running nf-core pipelines with Singularity containers; Singularity images will be downloaded automatically before execution of the pipeline. + +Please read the **Configure Singularity for use with Nextflow and nf-core** sub-section below. + + +## Running Nextflow with nf-core Pipelines on ShARC + +Nextflow is not currently available on ShARC as an environmental software module. The most simple solution to this issue is to install Nextflow and nf-core using a personal install of miniconda. This guide will describe the main steps, which are to: + +1. Install miniconda as a personal software module +2. Load and configure conda +3. Install Nextflow and nf-core within a conda environment +4. Configure Singularity for use with Nextflow and nf-core +5. Setup your project directory and configure your run +6. Submit your run to the SGE scheduler + + +### 1. Install Miniconda as a Personal Software Module + +Connect to ShARC via SSH and login to a worker node via an interactive session. + +```shell +# login +ssh -X username@sharc.shef.ac.uk + +# request a command line only interactive session - some extra resources prevent issues building conda env later +qrsh -l rmem=4G -pe smp 2 +``` + +Navigate your folder within the data area of the file store. + +```shell +cd /data/$USER +``` + +Download and run the miniconda installer by running the following series of commands: + +```shell +# download the latest installer file +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh + +# check the hashes match +sha256sum Miniconda3-latest-Linux-x86_64.sh + +# make the file executable +chmod +x Miniconda3-latest-Linux-x86_64.sh + +# run the installer +bash Miniconda3-latest-Linux-x86_64.sh +``` + +The miniconda installer will now run and walk you through the install process. There are two **IMPORTANT** things you must take note of at this point: + +1. You will have to overwrite the default install path when prompted by the miniconda installer to check the install path - the directory to which the install is attempted should be `/data/$USER/miniconda`. + +``` +Miniconda3 will now be installed into this location: +///miniconda3 + + - Press ENTER to confirm the location + - Press CTRL-C to abort the installation + - Or specify a different location below + +[///miniconda3] >>> /data/$USER/miniconda +``` + +2. **DO NOT** initialize miniconda at the end of the install process when prompted as shown here: + +``` +Do you wish the installer to initialize Miniconda3 +by running conda init? [yes|no] +[yes] >>> no +``` + +Once the installer has run, delete the installation script. + +```shell +rm Miniconda3-latest-Linux-x86_64.sh +``` + +Now make a modules folder and module file. + +```shell +# modules folder +mkdir /home/$USER/modules + +# module file +nano /home/$USER/modules/miniconda +``` + +Paste the below into the nano editor that opens upon running the final command. Note that this file is in Tcl not BASh, so environmental variable handing is different from the normal `$USER` for username. + +``` +#%Module10.2##################################################################### +## +## User Data Directory Miniconda module file +## +################################################################################ + +proc ModulesHelp { } { + global version + + puts stderr "Makes a user's personal install of Miniconda available." +} + +module-whatis "Makes a user's personal install of Miniconda available." + +# module variables + +set MINICONDA_DIR /data/$env(USER)/miniconda/bin + +prepend-path PATH $MINICONDA_DIR +``` + +Now run the following line to make your personal modules available for loading whenever you login. + +```shell +echo "module use /home/$USER/modules" >> ~/.bashrc +``` + +The last thing to note here is that you should not load the anaconda environmental module available to all HPC users and the personal miniconda module you have just made at the same time. + +For further information on making software available via a custom module file visit: + +[Making software available via a custom module file](https://docs.hpc.shef.ac.uk/en/latest/referenceinfo/environment-modules/creating-custom-modulefiles.html) + + +## 2. Load and Configure Conda + +Run the following commands in order and follow any prompts as appropriate: + +```shell +# load the miniconda module - if not already loaded +module load miniconda + +# disable base environment auto-activation +conda config --set auto_activate_base false + +# add the bioconda and conda-forge channels to conda configuration +conda config --add channels bioconda +conda config --add channels conda-forge + +# set channel_priority to "strict" +conda config --set channel_priority strict + +# ensure conda is up-to-date +conda update conda +``` + + +## 3. Install Nextflow and nf-core within a Conda Environment + +Run the following commands in order and follow any prompts as appropriate: + +```shell +# make the "nf_env" environment (in /home/$USER/.conda/envs/nf_env) +conda create --name nf_env nextflow nf-core + +# activate the environment +source activate nf_env + +# ensure all packages are up-to-date +conda update --all +``` + +You can now test the install has worked by running the following: + +```shell +# test the environment is working +nextflow info + +# test functionality +nextflow run hello +``` + +When you are finished, you can deactivate your conda environment using the command `conda deactivate`. + +Although you should not yet do this, should you wish to unload your personal miniconda module you can do so by running `module unload miniconda`. + +Step 5. describes the process of running an nf-core pipeline using Nextflow. You do not have to have a conda environment active for this part of the process as it will be loaded as part of your submission script, but you should not unload the miniconda module at this point. + + +## 4. Configure Singularity for use with Nextflow and nf-core + +When you run nextflow for the first time, Singularity will create a hidden directory `.singularity` in your `$HOME` directory `/home/$USER` which has very very limited (10GB) space available. It is therefore a good idea to create a directory somewhere else (e.g., `/data/$USER`) with more room and link the locations. To do this, run the following series of commands: + +```shell +# change directory to $HOME +cd $HOME + +# make the directory that will be linked to +mkdir /data/$USER/.singularity + +# link the new directory with the existing one +ln -s /data/$USER/.singularity .singularity +``` + + +## 5. Setup your Project and Configure your Run + +Whichever file store you decide to locate your project root directory in, the assumed project sub-directory structure within this guide is as follows: + +``` +/filestore/$USER/ +│ +└── project_root/ + │ + ├── config + ├── params + ├── sample_sheet + └── script +``` + +There are three things you will require to run an nf-core pipeline: + +1. A sample sheet +2. A pipeline launcher parameter configuration file +3. A submission script + +You can find nf-core pipelines by visiting [https://nf-co.re/pipelines](https://nf-co.re/pipelines). Each pipeline page has more information on how to use the pipeline as well as a full description of sample sheet requirements and formatting. + +Your sample sheet should be located inside your `sample_sheet` sub-directory. + +The general launch command in the script template below assumes you have configured your specific run using an nf-core pipeline launcher. For example, the launcher for the nf-core/rnaseq pipeline that can be found [here](https://nf-co.re/launch?pipeline=rnaseq). The parameters specified for your run using the launcher should be saved in a file named `nf-params.json` within the `params` sub-directory of your project root. + +To create your run script, navigate to the `script` sub-directory and run the following: + +```shell +nano nf_submission.sh +``` + +Paste the below into the editor ensuring to change the generic information for your own where indicated in the comment lines: + +```shell +#!/bin/bash + +## SGE scheduler flags + +# job name >>> edit "pipeline_name" for the name of the pipeline you are running e.g. rnaseq <<< +#$ -N nf-pipeline_name + +# specify queue and project for the nextflow driver job >>> keep and edit if using a priority queue else delete both <<< +#$ -q queue_name.q +#$ -P queue_name + +# request resources for the nextflow driver job +#$ -pe smp 1 +#$ -l rmem=2G + +# export environmental variables in current shell environment to job +#$ -V + +# send email >>> edit "username" <<< +#$ -M username@sheffield.ac.uk +#$ -m beas + +# merge standard error stream into the standard output stream +#$ -j y + +# output log file +#$ -o nextflow.log + + +## load miniconda module and activate analysis environment + +module load miniconda +source activate nf_env + + +## define and export variables + +# prevent java vm requesting too much memory and killing run +export NXF_OPTS="-Xms1g -Xmx2g" + +# path to singularity cache +export NXF_SINGULARITY_CACHEDIR="/home/$USER/.singularity" + +# project name >>> edit "project_name" so that it is the name of your project root directory <<< +PROJECT="project_name" + +# project directories >>> edit the name of the "filestore" e.g. fastdata <<< +PARAM_DIR="/filestore/$USER/$PROJECT/params" +CONFIG_DIR="/filestore/$USER/$PROJECT/config" + + +## run command >>> edit "pipeline" and "version" <<< + +nextflow run nf-core/pipeline \ +-r version \ +-profile sbc_sharc \ +-resume \ +-params-file ${PARAM_DIR}/nf-params.json + +``` + +Now save and exit by typing "Ctrl + O" then "Return" then "Ctrl + X". + +**OPTIONAL:** If you have specified a priority access queue in your submission script, you will need a personal configuration to send your jobs and not just your driver script to the appropriate queue. Navigate to the `config` sub-directory of your project folder and run the following: + +```shell +nano personal.config +``` + +Then paste the following into the editor, ensuring you enter the correct queue name: + +``` +process { + queue = 'queue-name.q' + clusterOptions = { "-P queue-name -l rmem=${task.memory.toGiga()}G" } +} +``` + +Save and exit by typing "Ctrl + O" then "Return" then "Ctrl + X". + +Now append `-c ${CONFIG_DIR}/personal.config` to the `nextflow run` command on a new line in your submission script. + + +## 6. Submit your Run to the SGE Scheduler + +Once you have fulfilled all of the requirements above, you should be ready to submit your batch job to the SGE scheduler on ShARC. From the project root, type the following: + +```bash +qsub ./scripts/nf_submission.sh +``` + +Your pipeline run should start momentarily. Good Luck! + diff --git a/nfcore_custom.config b/nfcore_custom.config index fae764b..abf163b 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -65,6 +65,7 @@ profiles { sage { includeConfig "${params.custom_config_base}/conf/sage.config" } sahmri { includeConfig "${params.custom_config_base}/conf/sahmri.config" } sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} + sbc_sharc { includeConfig "${params.custom_config_base}/conf/sbc_sharc.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } unibe_ibu { includeConfig "${params.custom_config_base}/conf/unibe_ibu.config" } diff --git a/pipeline/atacseq.config b/pipeline/atacseq.config new file mode 100644 index 0000000..f205f62 --- /dev/null +++ b/pipeline/atacseq.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/atacseq custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/atacseq folder and imported + * under a profile name here. + */ + +profiles { + sbc_sharc { includeConfig "${params.custom_config_base}/conf/pipeline/atacseq/sbc_sharc.config" } +} diff --git a/pipeline/chipseq.config b/pipeline/chipseq.config new file mode 100644 index 0000000..242aa92 --- /dev/null +++ b/pipeline/chipseq.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/chipseq custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/chipseq folder and imported + * under a profile name here. + */ + +profiles { + sbc_sharc { includeConfig "${params.custom_config_base}/conf/pipeline/chipseq/sbc_sharc.config" } +} diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config index 0486d86..b1d470f 100644 --- a/pipeline/rnaseq.config +++ b/pipeline/rnaseq.config @@ -11,5 +11,6 @@ profiles { eddie { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/eddie.config" } mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/mpcdf.config" } + sbc_sharc { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/sbc_sharc.config" } utd_sysbio { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/utd_sysbio.config" } } diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 12676b2..3c087aa 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -15,5 +15,6 @@ profiles { eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" } munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } + sbc_sharc { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/sbc_sharc.config" } uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" } } From 9fef35d7d3b4153eeade3af17fc08769bab000fb Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 13 Sep 2022 17:08:56 +0100 Subject: [PATCH 26/94] Updated pipeline configs for atacseq and chipseq --- conf/pipeline/atacseq/sbc_sharc.config | 2 +- conf/pipeline/chipseq/sbc_sharc.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 2e987d2..e50695c 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -11,7 +11,7 @@ */ -// process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config +// process-specific resource requirements - reduced specification from those in atacseq/conf/base.config process { diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 2741453..60912f3 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -11,7 +11,7 @@ */ -// process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config +// process-specific resource requirements - reduced specification from those in chipseq/conf/base.config process { From 61ae4636061d4f1296f363cc0929fb5e301bc5c7 Mon Sep 17 00:00:00 2001 From: Lewis Quayle <90088916+lquayle88@users.noreply.github.com> Date: Tue, 13 Sep 2022 17:13:57 +0100 Subject: [PATCH 27/94] Updated sbc_sharc.md --- docs/sbc_sharc.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/sbc_sharc.md b/docs/sbc_sharc.md index 7b055d1..7d5bcd3 100644 --- a/docs/sbc_sharc.md +++ b/docs/sbc_sharc.md @@ -2,17 +2,17 @@ The following nf-core pipelines have been successfully configured for use on the the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html): -- [atacseq](https://nf-co.re/atacseq) -- [chipseq](https://nf-co.re/chipseq) -- [rnaseq](https://nf-co.re/rnaseq) -- [sarek](https://nf-co.re/sarek) +- [nf-co.re/atacseq](https://nf-co.re/atacseq) +- [nf-co.re/chipseq](https://nf-co.re/chipseq) +- [nf-co.re/rnaseq](https://nf-co.re/rnaseq) +- [nf-co.re/sarek](https://nf-co.re/sarek) When using [`sbc_sharc.config`](../conf/sbc_sharc.config) with the pipelines listed above, the appropriate configuration file from the list below will be loaded automatically: -- atacseq: [sbc_sharc atacseq config](../conf/pipeline/atacseq/sbc_sharc.config) -- chipseq: [sbc_sharc chipseq config](../conf/pipeline/chipseq/sbc_sharc.config) -- rnaseq: [sbc_sharc rnaseq config](../conf/pipeline/rnaseq/sbc_sharc.config) -- sarek: [sbc_sharc sarek config](../conf/pipeline/sarek/sbc_sharc.config) +- [atacseq sbc_sharc.config](../conf/pipeline/atacseq/sbc_sharc.config) +- [chipseq sbc_sharc.config](../conf/pipeline/chipseq/sbc_sharc.config) +- [rnaseq sbc_sharc.config](../conf/pipeline/rnaseq/sbc_sharc.config) +- [sarek sbc_sharc.config](../conf/pipeline/sarek/sbc_sharc.config) The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file might work with other nf-core pipelines as it stands but we cannot guarantee they will run without issue. We will be continuing to create, test and optimise configurations for new pipelines in the future. From 3d3ee391beac2d24da61f50d29d38c7295a0bdac Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Tue, 13 Sep 2022 13:15:30 -0700 Subject: [PATCH 28/94] Limit the number of parallel transfers --- conf/sage.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/sage.config b/conf/sage.config index d43e719..4692ed2 100644 --- a/conf/sage.config +++ b/conf/sage.config @@ -30,6 +30,9 @@ aws { client { uploadChunkSize = 209715200 } + batch { + maxParallelTransfers = 1 + } } executor { name = 'awsbatch' From 91bfa7341f6a79e36eb8610350230ab2c92d4cef Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Wed, 14 Sep 2022 08:42:40 +0100 Subject: [PATCH 29/94] Made changes requested for PR --- README.md | 7 + conf/sbc_sharc.config | 2 - docs/pipeline/atacseq/sbc_sharc.md | 11 + docs/pipeline/chipseq/sbc_sharc.md | 11 + docs/pipeline/rnaseq/sbc_sharc.md | 11 + docs/pipeline/sarek/sbc_sharc.md | 11 + docs/sbc_sharc.md | 326 +---------------------------- 7 files changed, 58 insertions(+), 321 deletions(-) create mode 100644 docs/pipeline/atacseq/sbc_sharc.md create mode 100644 docs/pipeline/chipseq/sbc_sharc.md create mode 100644 docs/pipeline/rnaseq/sbc_sharc.md create mode 100644 docs/pipeline/sarek/sbc_sharc.md diff --git a/README.md b/README.md index 0cad49f..4544b05 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,10 @@ Currently documentation is available for the following pipelines within specific - ampliseq - [BINAC](docs/pipeline/ampliseq/binac.md) - [UPPMAX](docs/pipeline/ampliseq/uppmax.md) +- atacseq + - [SBC_SHARC](docs/pipeline/atacseq/sbc_sharc.md) +- chipseq + - [SBC_SHARC](docs/pipeline/chipseq/sbc_sharc.md) - eager - [EVA](docs/pipeline/eager/eva.md) - mag @@ -205,11 +209,14 @@ Currently documentation is available for the following pipelines within specific - rnafusion - [HASTA](docs/pipeline/rnafusion/hasta.md) - [MUNIN](docs/pipeline/rnafusion/munin.md) +- rnaseq + - [SBC_SHARC](docs/pipeline/rnaseq/sbc_sharc.md) - rnavar - [MUNIN](docs/pipeline/rnavar/munin.md) - sarek - [Cancer Research UK Manchester Institute](docs/pipeline/sarek/crukmi.md) - [MUNIN](docs/pipeline/sarek/munin.md) + - [SBC_SHARC](docs/pipeline/sarek/sbc_sharc.md) - [UPPMAX](docs/pipeline/sarek/uppmax.md) - taxprofiler - [EVA](docs/pipeline/taxprofiler/eva.md) diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index ca41185..20b8661 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -34,7 +34,6 @@ singularity { enabled = true autoMounts = true - // cacheDir = '////' } @@ -56,4 +55,3 @@ process { maxRetries = 2 } - diff --git a/docs/pipeline/atacseq/sbc_sharc.md b/docs/pipeline/atacseq/sbc_sharc.md new file mode 100644 index 0000000..f73d79b --- /dev/null +++ b/docs/pipeline/atacseq/sbc_sharc.md @@ -0,0 +1,11 @@ +# nf-core/configs: ATAC-Seq Specific Configuration - Sheffield Bioinformatics Core Facility ShARC + +Specific configuration for [nf-co.re/atacseq](https://nf-co.re/atacseq) pipeline + +## Usage + +To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). + +This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/atacseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. + +Example: `nextflow run nf-core/atacseq -profile sbc_sharc` diff --git a/docs/pipeline/chipseq/sbc_sharc.md b/docs/pipeline/chipseq/sbc_sharc.md new file mode 100644 index 0000000..31baba1 --- /dev/null +++ b/docs/pipeline/chipseq/sbc_sharc.md @@ -0,0 +1,11 @@ +# nf-core/configs: ChIP-Seq Specific Configuration - Sheffield Bioinformatics Core Facility ShARC + +Specific configuration for [nf-co.re/chipseq](https://nf-co.re/chipseq) pipeline + +## Usage + +To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). + +This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/chipseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. + +Example: `nextflow run nf-core/chipseq -profile sbc_sharc` diff --git a/docs/pipeline/rnaseq/sbc_sharc.md b/docs/pipeline/rnaseq/sbc_sharc.md new file mode 100644 index 0000000..562f84d --- /dev/null +++ b/docs/pipeline/rnaseq/sbc_sharc.md @@ -0,0 +1,11 @@ +# nf-core/configs: RNA-Seq Specific Configuration - Sheffield Bioinformatics Core Facility ShARC + +Specific configuration for [nf-co.re/rnaseq](https://nf-co.re/rnaseq) pipeline + +## Usage + +To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). + +This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/rnaseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. + +Example: `nextflow run nf-core/rnaseq -profile sbc_sharc` diff --git a/docs/pipeline/sarek/sbc_sharc.md b/docs/pipeline/sarek/sbc_sharc.md new file mode 100644 index 0000000..361be18 --- /dev/null +++ b/docs/pipeline/sarek/sbc_sharc.md @@ -0,0 +1,11 @@ +# nf-core/configs: Sarek Specific Configuration - Sheffield Bioinformatics Core Facility ShARC + +Specific configuration for [nf-co.re/sarek](https://nf-co.re/sarek) pipeline + +## Usage + +To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). + +This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/sarek/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. + +Example: `nextflow run nf-core/sarek -profile sbc_sharc` diff --git a/docs/sbc_sharc.md b/docs/sbc_sharc.md index 7d5bcd3..a40b7ea 100644 --- a/docs/sbc_sharc.md +++ b/docs/sbc_sharc.md @@ -1,5 +1,12 @@ # nf-core/configs: Sheffield Bioinformatics Core Facility ShARC Configuration + +## Using the SBC_ShARC Institutional Configuration Profile + +To use [`sbc_sharc.config`](../conf/sbc_sharc.config), run nextflow with an nf-core pipeline using `-profile sbc_sharc` (note the single hyphen). + +This will download and launch [`sbc_sharc.config`](../conf/sbc_sharc.config) which has been pre-configured with a setup suitable for the ShARC cluster and will automatically load the appropriate pipeline-specific configuration file. + The following nf-core pipelines have been successfully configured for use on the the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html): - [nf-co.re/atacseq](https://nf-co.re/atacseq) @@ -17,199 +24,10 @@ When using [`sbc_sharc.config`](../conf/sbc_sharc.config) with the pipelines lis The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file might work with other nf-core pipelines as it stands but we cannot guarantee they will run without issue. We will be continuing to create, test and optimise configurations for new pipelines in the future. -## Using the SBC_ShARC Institutional Configuration Profile - -To use [`sbc_sharc.config`](../conf/sbc_sharc.config), run nextflow with an nf-core pipeline using `-profile sbc_sharc` (note the single hyphen). This will download and launch [`sbc_sharc.config`](../conf/sbc_sharc.config) which has been pre-configured with a setup suitable for the ShARC cluster and will automatically load the appropriate pipeline-specific configuration file. - -For a full guide on how to setup and run Nextflow using nf-core pipelines on ShARC, see the **Running Nextflow with nf-core Pipelines on ShARC** section below. - - ## A Note on Singularity Containers The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file supports running nf-core pipelines with Singularity containers; Singularity images will be downloaded automatically before execution of the pipeline. -Please read the **Configure Singularity for use with Nextflow and nf-core** sub-section below. - - -## Running Nextflow with nf-core Pipelines on ShARC - -Nextflow is not currently available on ShARC as an environmental software module. The most simple solution to this issue is to install Nextflow and nf-core using a personal install of miniconda. This guide will describe the main steps, which are to: - -1. Install miniconda as a personal software module -2. Load and configure conda -3. Install Nextflow and nf-core within a conda environment -4. Configure Singularity for use with Nextflow and nf-core -5. Setup your project directory and configure your run -6. Submit your run to the SGE scheduler - - -### 1. Install Miniconda as a Personal Software Module - -Connect to ShARC via SSH and login to a worker node via an interactive session. - -```shell -# login -ssh -X username@sharc.shef.ac.uk - -# request a command line only interactive session - some extra resources prevent issues building conda env later -qrsh -l rmem=4G -pe smp 2 -``` - -Navigate your folder within the data area of the file store. - -```shell -cd /data/$USER -``` - -Download and run the miniconda installer by running the following series of commands: - -```shell -# download the latest installer file -wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh - -# check the hashes match -sha256sum Miniconda3-latest-Linux-x86_64.sh - -# make the file executable -chmod +x Miniconda3-latest-Linux-x86_64.sh - -# run the installer -bash Miniconda3-latest-Linux-x86_64.sh -``` - -The miniconda installer will now run and walk you through the install process. There are two **IMPORTANT** things you must take note of at this point: - -1. You will have to overwrite the default install path when prompted by the miniconda installer to check the install path - the directory to which the install is attempted should be `/data/$USER/miniconda`. - -``` -Miniconda3 will now be installed into this location: -///miniconda3 - - - Press ENTER to confirm the location - - Press CTRL-C to abort the installation - - Or specify a different location below - -[///miniconda3] >>> /data/$USER/miniconda -``` - -2. **DO NOT** initialize miniconda at the end of the install process when prompted as shown here: - -``` -Do you wish the installer to initialize Miniconda3 -by running conda init? [yes|no] -[yes] >>> no -``` - -Once the installer has run, delete the installation script. - -```shell -rm Miniconda3-latest-Linux-x86_64.sh -``` - -Now make a modules folder and module file. - -```shell -# modules folder -mkdir /home/$USER/modules - -# module file -nano /home/$USER/modules/miniconda -``` - -Paste the below into the nano editor that opens upon running the final command. Note that this file is in Tcl not BASh, so environmental variable handing is different from the normal `$USER` for username. - -``` -#%Module10.2##################################################################### -## -## User Data Directory Miniconda module file -## -################################################################################ - -proc ModulesHelp { } { - global version - - puts stderr "Makes a user's personal install of Miniconda available." -} - -module-whatis "Makes a user's personal install of Miniconda available." - -# module variables - -set MINICONDA_DIR /data/$env(USER)/miniconda/bin - -prepend-path PATH $MINICONDA_DIR -``` - -Now run the following line to make your personal modules available for loading whenever you login. - -```shell -echo "module use /home/$USER/modules" >> ~/.bashrc -``` - -The last thing to note here is that you should not load the anaconda environmental module available to all HPC users and the personal miniconda module you have just made at the same time. - -For further information on making software available via a custom module file visit: - -[Making software available via a custom module file](https://docs.hpc.shef.ac.uk/en/latest/referenceinfo/environment-modules/creating-custom-modulefiles.html) - - -## 2. Load and Configure Conda - -Run the following commands in order and follow any prompts as appropriate: - -```shell -# load the miniconda module - if not already loaded -module load miniconda - -# disable base environment auto-activation -conda config --set auto_activate_base false - -# add the bioconda and conda-forge channels to conda configuration -conda config --add channels bioconda -conda config --add channels conda-forge - -# set channel_priority to "strict" -conda config --set channel_priority strict - -# ensure conda is up-to-date -conda update conda -``` - - -## 3. Install Nextflow and nf-core within a Conda Environment - -Run the following commands in order and follow any prompts as appropriate: - -```shell -# make the "nf_env" environment (in /home/$USER/.conda/envs/nf_env) -conda create --name nf_env nextflow nf-core - -# activate the environment -source activate nf_env - -# ensure all packages are up-to-date -conda update --all -``` - -You can now test the install has worked by running the following: - -```shell -# test the environment is working -nextflow info - -# test functionality -nextflow run hello -``` - -When you are finished, you can deactivate your conda environment using the command `conda deactivate`. - -Although you should not yet do this, should you wish to unload your personal miniconda module you can do so by running `module unload miniconda`. - -Step 5. describes the process of running an nf-core pipeline using Nextflow. You do not have to have a conda environment active for this part of the process as it will be loaded as part of your submission script, but you should not unload the miniconda module at this point. - - -## 4. Configure Singularity for use with Nextflow and nf-core - When you run nextflow for the first time, Singularity will create a hidden directory `.singularity` in your `$HOME` directory `/home/$USER` which has very very limited (10GB) space available. It is therefore a good idea to create a directory somewhere else (e.g., `/data/$USER`) with more room and link the locations. To do this, run the following series of commands: ```shell @@ -223,133 +41,3 @@ mkdir /data/$USER/.singularity ln -s /data/$USER/.singularity .singularity ``` - -## 5. Setup your Project and Configure your Run - -Whichever file store you decide to locate your project root directory in, the assumed project sub-directory structure within this guide is as follows: - -``` -/filestore/$USER/ -│ -└── project_root/ - │ - ├── config - ├── params - ├── sample_sheet - └── script -``` - -There are three things you will require to run an nf-core pipeline: - -1. A sample sheet -2. A pipeline launcher parameter configuration file -3. A submission script - -You can find nf-core pipelines by visiting [https://nf-co.re/pipelines](https://nf-co.re/pipelines). Each pipeline page has more information on how to use the pipeline as well as a full description of sample sheet requirements and formatting. - -Your sample sheet should be located inside your `sample_sheet` sub-directory. - -The general launch command in the script template below assumes you have configured your specific run using an nf-core pipeline launcher. For example, the launcher for the nf-core/rnaseq pipeline that can be found [here](https://nf-co.re/launch?pipeline=rnaseq). The parameters specified for your run using the launcher should be saved in a file named `nf-params.json` within the `params` sub-directory of your project root. - -To create your run script, navigate to the `script` sub-directory and run the following: - -```shell -nano nf_submission.sh -``` - -Paste the below into the editor ensuring to change the generic information for your own where indicated in the comment lines: - -```shell -#!/bin/bash - -## SGE scheduler flags - -# job name >>> edit "pipeline_name" for the name of the pipeline you are running e.g. rnaseq <<< -#$ -N nf-pipeline_name - -# specify queue and project for the nextflow driver job >>> keep and edit if using a priority queue else delete both <<< -#$ -q queue_name.q -#$ -P queue_name - -# request resources for the nextflow driver job -#$ -pe smp 1 -#$ -l rmem=2G - -# export environmental variables in current shell environment to job -#$ -V - -# send email >>> edit "username" <<< -#$ -M username@sheffield.ac.uk -#$ -m beas - -# merge standard error stream into the standard output stream -#$ -j y - -# output log file -#$ -o nextflow.log - - -## load miniconda module and activate analysis environment - -module load miniconda -source activate nf_env - - -## define and export variables - -# prevent java vm requesting too much memory and killing run -export NXF_OPTS="-Xms1g -Xmx2g" - -# path to singularity cache -export NXF_SINGULARITY_CACHEDIR="/home/$USER/.singularity" - -# project name >>> edit "project_name" so that it is the name of your project root directory <<< -PROJECT="project_name" - -# project directories >>> edit the name of the "filestore" e.g. fastdata <<< -PARAM_DIR="/filestore/$USER/$PROJECT/params" -CONFIG_DIR="/filestore/$USER/$PROJECT/config" - - -## run command >>> edit "pipeline" and "version" <<< - -nextflow run nf-core/pipeline \ --r version \ --profile sbc_sharc \ --resume \ --params-file ${PARAM_DIR}/nf-params.json - -``` - -Now save and exit by typing "Ctrl + O" then "Return" then "Ctrl + X". - -**OPTIONAL:** If you have specified a priority access queue in your submission script, you will need a personal configuration to send your jobs and not just your driver script to the appropriate queue. Navigate to the `config` sub-directory of your project folder and run the following: - -```shell -nano personal.config -``` - -Then paste the following into the editor, ensuring you enter the correct queue name: - -``` -process { - queue = 'queue-name.q' - clusterOptions = { "-P queue-name -l rmem=${task.memory.toGiga()}G" } -} -``` - -Save and exit by typing "Ctrl + O" then "Return" then "Ctrl + X". - -Now append `-c ${CONFIG_DIR}/personal.config` to the `nextflow run` command on a new line in your submission script. - - -## 6. Submit your Run to the SGE Scheduler - -Once you have fulfilled all of the requirements above, you should be ready to submit your batch job to the SGE scheduler on ShARC. From the project root, type the following: - -```bash -qsub ./scripts/nf_submission.sh -``` - -Your pipeline run should start momentarily. Good Luck! - From 32403b6222676adf4fd0b5d8b7f7d27478d028d2 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Wed, 14 Sep 2022 08:46:11 +0100 Subject: [PATCH 30/94] Updated pipeline config docs for atacseq, chipseq and rnaseq --- docs/pipeline/atacseq/sbc_sharc.md | 2 +- docs/pipeline/chipseq/sbc_sharc.md | 2 +- docs/pipeline/rnaseq/sbc_sharc.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/pipeline/atacseq/sbc_sharc.md b/docs/pipeline/atacseq/sbc_sharc.md index f73d79b..1f33453 100644 --- a/docs/pipeline/atacseq/sbc_sharc.md +++ b/docs/pipeline/atacseq/sbc_sharc.md @@ -6,6 +6,6 @@ Specific configuration for [nf-co.re/atacseq](https://nf-co.re/atacseq) pipeline To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). -This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/atacseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. +This will download and launch the atacseq specific [`sbc_sharc.config`](../../../conf/pipeline/atacseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. Example: `nextflow run nf-core/atacseq -profile sbc_sharc` diff --git a/docs/pipeline/chipseq/sbc_sharc.md b/docs/pipeline/chipseq/sbc_sharc.md index 31baba1..4280db9 100644 --- a/docs/pipeline/chipseq/sbc_sharc.md +++ b/docs/pipeline/chipseq/sbc_sharc.md @@ -6,6 +6,6 @@ Specific configuration for [nf-co.re/chipseq](https://nf-co.re/chipseq) pipeline To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). -This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/chipseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. +This will download and launch the chipseq specific [`sbc_sharc.config`](../../../conf/pipeline/chipseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. Example: `nextflow run nf-core/chipseq -profile sbc_sharc` diff --git a/docs/pipeline/rnaseq/sbc_sharc.md b/docs/pipeline/rnaseq/sbc_sharc.md index 562f84d..d62fe25 100644 --- a/docs/pipeline/rnaseq/sbc_sharc.md +++ b/docs/pipeline/rnaseq/sbc_sharc.md @@ -6,6 +6,6 @@ Specific configuration for [nf-co.re/rnaseq](https://nf-co.re/rnaseq) pipeline To use, run nextflow with the pipeline using `-profile sbc_sharc` (note the single hyphen). -This will download and launch the sarek specific [`sbc_sharc.config`](../../../conf/pipeline/rnaseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. +This will download and launch the rnaseq specific [`sbc_sharc.config`](../../../conf/pipeline/rnaseq/sbc_sharc.config) which has been pre-configured with a setup suitable for the [University of Sheffield ShARC cluster](https://docs.hpc.shef.ac.uk/en/latest/index.html) and will automatically load the appropriate pipeline-specific configuration file. Example: `nextflow run nf-core/rnaseq -profile sbc_sharc` From be356eb400c8dce34c1638d5754b44ad7d167efa Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Wed, 14 Sep 2022 10:24:56 +0100 Subject: [PATCH 31/94] Ran prettier on docs/sbc_sharc.md --- docs/sbc_sharc.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/sbc_sharc.md b/docs/sbc_sharc.md index a40b7ea..f82b348 100644 --- a/docs/sbc_sharc.md +++ b/docs/sbc_sharc.md @@ -1,6 +1,5 @@ # nf-core/configs: Sheffield Bioinformatics Core Facility ShARC Configuration - ## Using the SBC_ShARC Institutional Configuration Profile To use [`sbc_sharc.config`](../conf/sbc_sharc.config), run nextflow with an nf-core pipeline using `-profile sbc_sharc` (note the single hyphen). @@ -23,7 +22,6 @@ When using [`sbc_sharc.config`](../conf/sbc_sharc.config) with the pipelines lis The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file might work with other nf-core pipelines as it stands but we cannot guarantee they will run without issue. We will be continuing to create, test and optimise configurations for new pipelines in the future. - ## A Note on Singularity Containers The [`sbc_sharc.config`](../conf/sbc_sharc.config) configuration file supports running nf-core pipelines with Singularity containers; Singularity images will be downloaded automatically before execution of the pipeline. @@ -40,4 +38,3 @@ mkdir /data/$USER/.singularity # link the new directory with the existing one ln -s /data/$USER/.singularity .singularity ``` - From c0effc0366c0dce8815edc733b2143b5da1a19b6 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 14 Sep 2022 15:34:27 +0200 Subject: [PATCH 32/94] Add HKI profile files --- .github/workflows/main.yml | 1 + README.md | 1 + conf/hki.config | 104 +++++++++++++++++++++++++++++++++++++ docs/hki.md | 24 +++++++++ nfcore_custom.config | 1 + 5 files changed, 131 insertions(+) create mode 100644 conf/hki.config create mode 100644 docs/hki.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca01a15..56755f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,6 +62,7 @@ jobs: - "google" - "hasta" - "hebbe" + - "hki" - "icr_davros" - "ifb_core" - "imperial" diff --git a/README.md b/README.md index 4544b05..5264a3c 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Currently documentation is available for the following systems: - [GOOGLE](docs/google.md) - [HASTA](docs/hasta.md) - [HEBBE](docs/hebbe.md) +- [HKI](docs/hki.md) - [ICR_DAVROS](docs/icr_davros.md) - [IMPERIAL](docs/imperial.md) - [JAX](docs/jax.md) diff --git a/conf/hki.config b/conf/hki.config new file mode 100644 index 0000000..b73558f --- /dev/null +++ b/conf/hki.config @@ -0,0 +1,104 @@ +params { + config_profile_description = 'HKI clusters profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' +} + +profiles { + apate { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 128.GB + max_cpus = 32 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + aither { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 128.GB + max_cpus = 32 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + arges { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 64.GB + max_cpus = 12 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + debug { + cleanup = false + } +} diff --git a/docs/hki.md b/docs/hki.md new file mode 100644 index 0000000..cd102ce --- /dev/null +++ b/docs/hki.md @@ -0,0 +1,24 @@ +# nf-core/configs: HKI Configuration + +All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en/). + +To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluter. The number of parallel jobs that run is currently limited to 8. + +The profiles currently available are: + +- apate (uses singularity, cleanup set to true by default) +- arges (uses singularity, cleanup set to true by default) +- aither (uses singularity, cleanup set to true by default) +- debug (sets cleanup to false for debugging purposes, use e.g. `profile hki,,debug`) + +Note that Nextflow is not necessarily installed by default on the HKI HPC cluster(s). You will need to install it into a directory you have write access to. +Follow these instructions from the Nextflow documentation. + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline +has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. + +> NB: You will need an account to use the HKI HPC clusters in order to run the pipeline. If in doubt contact the ICT Service Desk. +> NB: Nextflow will need to submit the jobs via SLURM to the HKI HPC clusters and as such the commands above will have to be executed on the login +> node. If in doubt contact ICT. diff --git a/nfcore_custom.config b/nfcore_custom.config index abf163b..d93e266 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -44,6 +44,7 @@ profiles { google { includeConfig "${params.custom_config_base}/conf/google.config" } hasta { includeConfig "${params.custom_config_base}/conf/hasta.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } + hki { includeConfig "${params.custom_config_base}/conf/hki.config"} icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From b8751af6c60818102d8a1573016f866572754a5d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 14 Sep 2022 16:17:17 +0200 Subject: [PATCH 33/94] Update conf/pipeline/mag/eva.config --- conf/pipeline/mag/eva.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config index 37a531d..81d8d0c 100644 --- a/conf/pipeline/mag/eva.config +++ b/conf/pipeline/mag/eva.config @@ -4,11 +4,6 @@ params { config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' } -env { - OPENBLAS_NUM_THREADS=1 - OMP_NUM_THREADS=1 -} - process { withName: FASTQC { From ec638e62dece9fe6519da1a6feb9c1c2f0f517de Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Sep 2022 09:10:29 +0200 Subject: [PATCH 34/94] Apply suggestions from code review Co-authored-by: Jasmin F <73216762+jasmezz@users.noreply.github.com> --- conf/hki.config | 6 +++--- docs/hki.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/hki.config b/conf/hki.config index b73558f..63718bf 100644 --- a/conf/hki.config +++ b/conf/hki.config @@ -7,7 +7,7 @@ params { profiles { apate { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 128.GB @@ -38,7 +38,7 @@ profiles { aither { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'aither HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 128.GB @@ -69,7 +69,7 @@ profiles { arges { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'arges HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 64.GB diff --git a/docs/hki.md b/docs/hki.md index cd102ce..de45c20 100644 --- a/docs/hki.md +++ b/docs/hki.md @@ -1,10 +1,10 @@ # nf-core/configs: HKI Configuration -All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en/). +All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en). -To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluter. The number of parallel jobs that run is currently limited to 8. +To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluster. The number of parallel jobs that run is currently limited to 8. -The profiles currently available are: +The currently available profiles are: - apate (uses singularity, cleanup set to true by default) - arges (uses singularity, cleanup set to true by default) From 55971bb94de2194f3a3eea7c5965d5dde1b9fef5 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 16 Sep 2022 17:29:08 +0100 Subject: [PATCH 35/94] Modularised error handling strategy to per pipeline basis --- conf/pipeline/atacseq/sbc_sharc.config | 9 +++++++++ conf/pipeline/chipseq/sbc_sharc.config | 9 +++++++++ conf/pipeline/rnaseq/sbc_sharc.config | 8 ++++++++ conf/pipeline/sarek/sbc_sharc.config | 7 ++++++- conf/sbc_sharc.config | 22 +++++++++------------- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index e50695c..0660591 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -15,6 +15,14 @@ process { + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels + withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } @@ -72,3 +80,4 @@ def check_max(obj, type) { } } } + diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 60912f3..6fef3ba 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -14,6 +14,14 @@ // process-specific resource requirements - reduced specification from those in chipseq/conf/base.config process { + + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } @@ -72,3 +80,4 @@ def check_max(obj, type) { } } } + diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 52bf0ff..8cefa40 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -14,6 +14,14 @@ // process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config process { + + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 204d73b..434b8a2 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -14,8 +14,13 @@ // process-specific resource requirements - reduced specification from those in sarek/conf/base.config process { - + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } + maxRetries = 2 + + // process labels withLabel:process_low { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 20b8661..de1ab0c 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -28,16 +28,6 @@ params { } -// container engine - -singularity { - - enabled = true - autoMounts = true - -} - - // hpc configuration specific to ShARC process { @@ -49,9 +39,15 @@ process { queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } clusterOptions = { "-l rmem=${task.memory.toGiga()}G" } - // error and retry handling +} + - errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } - maxRetries = 2 +// container engine + +singularity { + + enabled = true + autoMounts = true } + From 8bb9361359a03125d69f49ab68344da4df2dd555 Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Sat, 17 Sep 2022 10:14:04 -0700 Subject: [PATCH 36/94] Improve reliability of file transfers --- conf/sage.config | 3 +++ docs/sage.md | 1 + 2 files changed, 4 insertions(+) diff --git a/conf/sage.config b/conf/sage.config index 4692ed2..c77f8fa 100644 --- a/conf/sage.config +++ b/conf/sage.config @@ -29,9 +29,12 @@ aws { region = "us-east-1" client { uploadChunkSize = 209715200 + uploadMaxThreads = 4 } batch { maxParallelTransfers = 1 + maxTransferAttempts = 5 + delayBetweenAttempts = '120 sec' } } executor { diff --git a/docs/sage.md b/docs/sage.md index 1e36fed..7b4a8fe 100644 --- a/docs/sage.md +++ b/docs/sage.md @@ -8,6 +8,7 @@ This global configuration includes the following tweaks: - Enable retries by default when exit codes relate to insufficient memory - Allow pending jobs to finish if the number of retries are exhausted - Increase the amount of time allowed for file transfers +- Improve reliability of file transfers with retries and reduced concurrency - Increase the default chunk size for multipart uploads to S3 - Slow down job submission rate to avoid overwhelming any APIs - Define the `check_max()` function, which is missing in Sarek v2 From c709be3b599d463fcfa82196fd4c9c5fa1e99513 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Sun, 18 Sep 2022 12:06:37 +0200 Subject: [PATCH 37/94] update cfc config maintenance --- conf/cfc.config | 2 +- conf/cfc_dev.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index 1948e14..999e8d6 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -12,7 +12,7 @@ singularity { process { executor = 'slurm' - queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + queue = 'qbic' scratch = 'true' } diff --git a/conf/cfc_dev.config b/conf/cfc_dev.config index 1d61baf..87caf66 100644 --- a/conf/cfc_dev.config +++ b/conf/cfc_dev.config @@ -11,7 +11,7 @@ singularity { process { executor = 'slurm' - queue = { task.memory > 60.GB || task.cpus > 20 ? 'qbic' : 'compute' } + queue = 'qbic' scratch = 'true' } @@ -25,4 +25,4 @@ params { max_memory = 1999.GB max_cpus = 128 max_time = 140.h -} \ No newline at end of file +} From adc88cfa1f3edec886261c5f721cb97581a019ac Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Mon, 19 Sep 2022 16:30:30 +0100 Subject: [PATCH 38/94] Added config for limiting queue size and job submit rate --- conf/sbc_sharc.config | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index de1ab0c..bf00d33 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -11,16 +11,21 @@ */ +// nf-core specific parameters displayed in header summary of each run + params { - - // nf-core specific parameters displayed in header summary of each run config_profile_description = 'Sheffield Bioinformatics Core - ShARC' config_profile_contact = 'Lewis Quayle (l.quayle@sheffield.ac.uk)' config_profile_url = 'https://docs.hpc.shef.ac.uk/en/latest/sharc/index.html' - - // hpc resource limits +} + + +// hpc resource limits + +params { + max_cpus = 16 max_memory = 64.GB max_time = 96.h @@ -42,6 +47,16 @@ process { } +// optional executor settings + +executor { + + queueSize = 10 + submitRateLimit = '1 sec' + +} + + // container engine singularity { @@ -50,4 +65,3 @@ singularity { autoMounts = true } - From 1471257aaf2f6253b97bc3133c18f990e96dfb35 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 20 Sep 2022 19:51:31 +0100 Subject: [PATCH 39/94] Modified config for job submit rate --- conf/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index bf00d33..a8a096e 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -52,7 +52,7 @@ process { executor { queueSize = 10 - submitRateLimit = '1 sec' + submitRateLimit = '12/1min' } From 2e9578498aae79e6fdbdd67ddc59c0b04ef94868 Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:20:42 +0200 Subject: [PATCH 40/94] add ku_sund_dangpu.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index d93e266..f777280 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -49,6 +49,7 @@ profiles { ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } jax { includeConfig "${params.custom_config_base}/conf/jax.config" } + ku_sund_sdangpu {includeConfig "${params.custom_config_base}/conf/ku_sund_dangpu.config"} lugh { includeConfig "${params.custom_config_base}/conf/lugh.config" } maestro { includeConfig "${params.custom_config_base}/conf/maestro.config" } marvin { includeConfig "${params.custom_config_base}/conf/marvin.config" } From ce916bf4b56255174afc3b4bc76182ba4806d61d Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:21:10 +0200 Subject: [PATCH 41/94] fix typo --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index f777280..aba5349 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -49,7 +49,7 @@ profiles { ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } jax { includeConfig "${params.custom_config_base}/conf/jax.config" } - ku_sund_sdangpu {includeConfig "${params.custom_config_base}/conf/ku_sund_dangpu.config"} + ku_sund_dangpu {includeConfig "${params.custom_config_base}/conf/ku_sund_dangpu.config"} lugh { includeConfig "${params.custom_config_base}/conf/lugh.config" } maestro { includeConfig "${params.custom_config_base}/conf/maestro.config" } marvin { includeConfig "${params.custom_config_base}/conf/marvin.config" } From 5315ddd92f207528ab512dedac78d0e6b1fb685e Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:22:26 +0200 Subject: [PATCH 42/94] add KU_SUND_DANGPU --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5264a3c..8eee40a 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ Currently documentation is available for the following systems: - [ICR_DAVROS](docs/icr_davros.md) - [IMPERIAL](docs/imperial.md) - [JAX](docs/jax.md) +- KU_SUND_DANGPU (docs/ku_sund_dangpu.md) - [LUGH](docs/lugh.md) - [MAESTRO](docs/maestro.md) - [MARVIN](docs/marvin.md) From 55d7b3fc000c228b90e4da4b4afbbec1fcfb98a4 Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:35:08 +0200 Subject: [PATCH 43/94] add KU SUND DANGPU add the profile for Dangpu server hosted by Novo Nordisk Foundation Center for Stem Cell Medicine (reNEW)/ Novo Nordisk Foundation Center for Protein Research (CPR) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8eee40a..e3cff9f 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Currently documentation is available for the following systems: - [ICR_DAVROS](docs/icr_davros.md) - [IMPERIAL](docs/imperial.md) - [JAX](docs/jax.md) -- KU_SUND_DANGPU (docs/ku_sund_dangpu.md) +- [KU SUND DANGPU] (docs/ku_sund_dangpu.md) - [LUGH](docs/lugh.md) - [MAESTRO](docs/maestro.md) - [MARVIN](docs/marvin.md) From 339a072884667fbe9dd5cc4784ad3d1356fa2ac0 Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:35:42 +0200 Subject: [PATCH 44/94] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e3cff9f..838d288 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ Currently documentation is available for the following systems: - [ICR_DAVROS](docs/icr_davros.md) - [IMPERIAL](docs/imperial.md) - [JAX](docs/jax.md) -- [KU SUND DANGPU] (docs/ku_sund_dangpu.md) +- [KU SUND DANGPU](docs/ku_sund_dangpu.md) - [LUGH](docs/lugh.md) - [MAESTRO](docs/maestro.md) - [MARVIN](docs/marvin.md) From 65b6bf7c623643d312c946e011eaf5a39baf25c8 Mon Sep 17 00:00:00 2001 From: Adrija Kalvisa Date: Thu, 22 Sep 2022 13:37:45 +0200 Subject: [PATCH 45/94] add ku_sund_dangpu --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56755f1..e4d68ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -67,6 +67,7 @@ jobs: - "ifb_core" - "imperial" - "jax" + - "ku_sund_dangpu" - "lugh" - "marvin" - "medair" From 798d845a0dd327b7ae4eb4c091588e463574fbcf Mon Sep 17 00:00:00 2001 From: AdrijaK Date: Thu, 22 Sep 2022 14:21:02 +0200 Subject: [PATCH 46/94] run prettier --- docs/ku_sund_dangpu.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ku_sund_dangpu.md b/docs/ku_sund_dangpu.md index aea1b40..72baf46 100644 --- a/docs/ku_sund_dangpu.md +++ b/docs/ku_sund_dangpu.md @@ -1,6 +1,6 @@ # nf-core/configs: ku_sund_dangpu configuration -All nf-core pipelines have been successfully configured for use on the DANGPU at the +All nf-core pipelines have been successfully configured for use on the DANGPU at the Novo Nordisk Foundation Center for Stem Cell Medicine (reNEW) and the Novo Nordisk Foundation Center for Protein Research (CPR) at the University of Copenhagen. To use, run the pipeline with `-profile ku_sund_dangpu`. This will download and launch the [`ku_sund_dangpu.config`](../conf/ku_sund_dangpu.config) which has been pre-configured with a setup suitable for the DANGPU. @@ -14,22 +14,22 @@ Before running the pipeline you will need to load Nextflow and Singularity using module purge module load java/11.0.15 nextflow/22.04.4 singularity/3.8.0 # alternative modules for older nextflow version (v.21) that works with java 8: -# module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 +# module load jdk/1.8.0_291 nextflow/21.04.1.5556 singularity/3.8.0 export NXF_OPTS='-Xms1g -Xmx4g' export NXF_HOME=/projects/dan1/people/${USER}/cache/nxf-home export NXF_TEMP=/scratch/tmp -export NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images +export NXF_SINGULARITY_CACHEDIR=/projects/dan1/people/${USER}/cache/singularity-images ``` Create the user-specific nextflow directories if they don't exist yet: + ``` mkdir $NXF_SINGULARITY_CACHEDIR mkdir $NXF_HOME ``` -Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm to be the resource manager within the `ku_sund_dangpu profile`. Just make sure that the pipeline is run within a tmux session. +Finally, download and test the pipeline of choice using the `-profile ku_sund_dangpu`. Note that normally you would run resource-intensive commands with slurm, but in case of nf-core pipelines you do not have to do this: we have pre-configured slurm to be the resource manager within the `ku_sund_dangpu profile`. Just make sure that the pipeline is run within a tmux session. ``` nextflow run nf-core/rnaseq -profile test,ku_sund_dangpu ``` - From 7e34beb3c9d75f580888b2b025aa684206df27ca Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 11:08:39 +0100 Subject: [PATCH 47/94] Updated resource requests for compatibility with general worker nodes --- conf/pipeline/atacseq/sbc_sharc.config | 2 +- conf/pipeline/chipseq/sbc_sharc.config | 2 +- conf/pipeline/rnaseq/sbc_sharc.config | 6 +----- conf/pipeline/sarek/sbc_sharc.config | 10 +++++----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 0660591..97cd758 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 6fef3ba..a2f6431 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8cefa40..8f905f7 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -45,10 +45,6 @@ process { time = { check_max( 12.h * task.attempt, 'time' ) } } - withLabel:process_high_memory { - memory = { check_max( 60.GB * task.attempt, 'memory' ) } - } - } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 434b8a2..24312a7 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +46,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 60.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } @@ -54,7 +54,7 @@ process { withName:'BWAMEM1_MEM|BWAMEM2_MEM' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -71,11 +71,11 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From 17c3767fef85f129eee9632ee6202822ad550ba8 Mon Sep 17 00:00:00 2001 From: Simon Pearce <24893913+SPPearce@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:53:54 +0100 Subject: [PATCH 48/94] Update crukmi.config Fix the singularity cache dir and add process_single. --- conf/crukmi.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/crukmi.config b/conf/crukmi.config index 4823585..778935b 100644 --- a/conf/crukmi.config +++ b/conf/crukmi.config @@ -5,11 +5,8 @@ params { config_profile_url = 'http://scicom.picr.man.ac.uk/projects/user-support/wiki' } -env { - SINGULARITY_CACHEDIR = '/lmod/nextflow_software' -} - singularity { + cacheDir = '/lmod/nextflow_software' enabled = true autoMounts = true } @@ -22,6 +19,11 @@ process { maxErrors = '-1' maxRetries = 3 + withLabel:process_single { + cpus = { check_max( 1 * task.attempt, 'cpus' ) } + memory = { check_max( 5.GB * task.attempt, 'memory' ) } + } + withLabel:process_low { cpus = { check_max( 1 * task.attempt, 'cpus' ) } memory = { check_max( 5.GB * task.attempt, 'memory' ) } From ab1ad832dfed33016503f7011a1ae4d732a68922 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 13:18:50 +0100 Subject: [PATCH 49/94] Updated resource requests for compatibility with high memory nodes --- conf/pipeline/atacseq/sbc_sharc.config | 12 ++++++------ conf/pipeline/chipseq/sbc_sharc.config | 12 ++++++------ conf/pipeline/rnaseq/sbc_sharc.config | 16 ++++++++++------ conf/pipeline/sarek/sbc_sharc.config | 24 ++++++++++++------------ conf/sbc_sharc.config | 4 ++-- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 97cd758..60cbc49 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index a2f6431..10de65b 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8f905f7..fe50bc6 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -45,6 +45,10 @@ process { time = { check_max( 12.h * task.attempt, 'time' ) } } + withLabel:process_high_memory { + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + } + } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 24312a7..75e2be0 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,20 +46,20 @@ process { } withLabel:process_high_memory { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } // process name withName:'BWAMEM1_MEM|BWAMEM2_MEM' { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FASTP' { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } } withName:'FASTQC|FASTP|MOSDEPTH|SAMTOOLS_CONVERT' { @@ -71,11 +71,11 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index a8a096e..21dd18b 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -27,7 +27,7 @@ params { params { max_cpus = 16 - max_memory = 64.GB + max_memory = 256.GB max_time = 96.h } @@ -52,7 +52,7 @@ process { executor { queueSize = 10 - submitRateLimit = '12/1min' + submitRateLimit = '1 sec' } From c42a5a9a94df4845630f68633c4401c8ce83269b Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 17:01:58 +0100 Subject: [PATCH 50/94] Updated resources requested by pipelines --- conf/pipeline/atacseq/sbc_sharc.config | 10 +++++----- conf/pipeline/chipseq/sbc_sharc.config | 10 +++++----- conf/pipeline/rnaseq/sbc_sharc.config | 10 +++++----- conf/pipeline/sarek/sbc_sharc.config | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 60cbc49..0660591 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 10de65b..6fef3ba 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index fe50bc6..8033609 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 75e2be0..44dc57d 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } + cpus = { check_max( 6 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -53,13 +53,13 @@ process { // process name withName:'BWAMEM1_MEM|BWAMEM2_MEM' { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FASTP' { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } } withName:'FASTQC|FASTP|MOSDEPTH|SAMTOOLS_CONVERT' { From 8103718389494dd91050ee160c1b7b4e2bd67749 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 30 Sep 2022 16:54:48 +0100 Subject: [PATCH 51/94] Reduced maxRetries upon error in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 44dc57d..95b3ee3 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -18,7 +18,7 @@ process { // error and retry handling errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } - maxRetries = 2 + maxRetries = 1 // process labels From dbb3b3b44b3205fb4c1c034da4d73159b346cf37 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 07:43:46 +0100 Subject: [PATCH 52/94] Reinstated maxRetries and increased memory for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 95b3ee3..4ddf2ba 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -18,7 +18,7 @@ process { // error and retry handling errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } - maxRetries = 1 + maxRetries = 2 // process labels @@ -75,7 +75,7 @@ process { } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 20.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From 10c14902650bcb7f448b660938581d6bc8ee5500 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 23:15:13 +0100 Subject: [PATCH 53/94] Increased memory and time for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 4ddf2ba..570284d 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -76,6 +76,7 @@ process { withName:'GATK4_MARKDUPLICATES' { memory = { check_max( 20.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From 9b451b53b7a539c98cf91d25642792318146b0c2 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 23:31:51 +0100 Subject: [PATCH 54/94] Increased cpus and memory for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 570284d..170b8bd 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -26,13 +26,13 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 6 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 6.h * task.attempt, 'time' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } } withLabel:process_high { @@ -75,7 +75,8 @@ process { } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 20.GB * task.attempt, 'memory' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 30.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From e7ac3ecd1934b8554ec28c02e00998e0e4869d22 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sun, 2 Oct 2022 00:24:39 +0100 Subject: [PATCH 55/94] Modified resource requests in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 170b8bd..d970574 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -76,7 +76,7 @@ process { withName:'GATK4_MARKDUPLICATES' { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 30.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From c479780d8f2e0e802b56c12ff836d11aa223397f Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:38:20 +0000 Subject: [PATCH 56/94] create mana config --- conf/mana.config | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 conf/mana.config diff --git a/conf/mana.config b/conf/mana.config new file mode 100644 index 0000000..47f1992 --- /dev/null +++ b/conf/mana.config @@ -0,0 +1,20 @@ +params { + config_profile_description = 'University of Hawaii at Manoa' + config_profile_url = 'http://www.hawaii.edu/its/ci/' + config_profile_contact = 'Cedric Arisdakessian' + + max_memory = 400.GB + max_cpus = 96 + max_time = 72.h +} + +process { + executor = 'slurm' + queue = 'shared,exclusive,kill-shared,kill-exclusive' + module = 'tools/Singularity' +} + +singularity { + enabled = true + cacheDir = "$HOME/.singularity_images_cache" +} From 32cdea4b59788dedee89fd88e0e4f799fb91cc20 Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:38:48 +0000 Subject: [PATCH 57/94] documentation for mana config --- docs/mana.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 docs/mana.md diff --git a/docs/mana.md b/docs/mana.md new file mode 100644 index 0000000..030749c --- /dev/null +++ b/docs/mana.md @@ -0,0 +1,46 @@ +# nf-core/configs Mana (at University of Hawaii at Manoa) Configuration + +To use, run the pipeline with `-profile mana`. It will use the following parameters for Mana (UHM HPCC): + +- Load singularity and use it as default container technology +- Setup a container cache directory in your home (~/.singularity_images_cache) +- Select appropriate queues (currently: `shared,exclusive,kill-shared,kill-exclusive`) +- Set the maximum available resources (available in 09/02/2022): + - CPUs: 96 + - Memory: 400.GB + - Time: 72.h + + +## Pre-requisites + +In order to run a nf-core pipeline on Mana, you will need to setup nextflow in your environment. +At the moment, nextflow is not available as a module (but might be in the future). + +### Install nextflow in a conda environment + +Before we start, we will need to work on an interactive node (currently, mana doesn't let you execute any program in the login node): + +```bash +# Request an interactive sandbox node for 30 min +srun --pty -t 30 -p sandbox /bin/bash +``` + +To setup nextflow on your account, follow these steps. + +```bash +# Load the latest anaconda3 module +module load lang/Anaconda3 + +# Initialize environment +. /opt/apps/software/lang/Anaconda3/5.1.0/etc/profile.d/conda.sh + +# Install nextflow (here in base environment, but you can create a new one if you'd like) +conda install -c bioconda nextflow +``` + +If you want these settings to be persistent, you can add the first 2 commands in your .bash_profile file like this: + +```bash +echo "module load lang/Anaconda3" >> ~/.bash_profile +echo ". /opt/apps/software/lang/Anaconda3/5.1.0/etc/profile.d/conda.sh >> ~/.bash_profile" +``` \ No newline at end of file From 9b710a07f54be03cb161fcda83b31c5a407d8d25 Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:41:23 +0000 Subject: [PATCH 58/94] add mana config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index aba5349..aec233f 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -52,6 +52,7 @@ profiles { ku_sund_dangpu {includeConfig "${params.custom_config_base}/conf/ku_sund_dangpu.config"} lugh { includeConfig "${params.custom_config_base}/conf/lugh.config" } maestro { includeConfig "${params.custom_config_base}/conf/maestro.config" } + mana { includeConfig "${params.custom_config_base}/conf/mana.config" } marvin { includeConfig "${params.custom_config_base}/conf/marvin.config" } medair { includeConfig "${params.custom_config_base}/conf/medair.config" } mjolnir_globe { includeConfig "${params.custom_config_base}/conf/mjolnir_globe.config" } From 7b1e360a3588627de30c76431d7a561d39b792da Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:42:07 +0000 Subject: [PATCH 59/94] added mana --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 838d288..e432ec7 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Currently documentation is available for the following systems: - [KU SUND DANGPU](docs/ku_sund_dangpu.md) - [LUGH](docs/lugh.md) - [MAESTRO](docs/maestro.md) +- [MANA](docs/mana.md) - [MARVIN](docs/marvin.md) - [MEDAIR](docs/medair.md) - [MJOLNIR_GLOBE](docs/mjolnir_globe.md) From 9cc679c811bda824f581cacbcdc764ee63b48642 Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:43:51 +0000 Subject: [PATCH 60/94] fix mana name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e432ec7..c476905 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Currently documentation is available for the following systems: - [KU SUND DANGPU](docs/ku_sund_dangpu.md) - [LUGH](docs/lugh.md) - [MAESTRO](docs/maestro.md) -- [MANA](docs/mana.md) +- [Mana](docs/mana.md) - [MARVIN](docs/marvin.md) - [MEDAIR](docs/medair.md) - [MJOLNIR_GLOBE](docs/mjolnir_globe.md) From 3b8b1702030e6e85e1ff147b0b7355de4b7b17f2 Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:47:03 +0000 Subject: [PATCH 61/94] added mana in gh actions --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4d68ca..16453da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,6 +69,7 @@ jobs: - "jax" - "ku_sund_dangpu" - "lugh" + - "mana" - "marvin" - "medair" - "mjolnir_globe" From e89cb7bd5e6257a8918bbfe427396cab8758c74d Mon Sep 17 00:00:00 2001 From: Cedric Arisdakessian Date: Mon, 3 Oct 2022 03:49:04 +0000 Subject: [PATCH 62/94] fix indentation --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16453da..c057f2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,7 +69,7 @@ jobs: - "jax" - "ku_sund_dangpu" - "lugh" - - "mana" + - "mana" - "marvin" - "medair" - "mjolnir_globe" From d429a59a0d76207dbff6be8c98984df4138372be Mon Sep 17 00:00:00 2001 From: Cedric Date: Sun, 2 Oct 2022 18:44:22 -1000 Subject: [PATCH 63/94] add automounts for singularity --- conf/mana.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/mana.config b/conf/mana.config index 47f1992..93d674c 100644 --- a/conf/mana.config +++ b/conf/mana.config @@ -17,4 +17,5 @@ process { singularity { enabled = true cacheDir = "$HOME/.singularity_images_cache" + autoMounts = true } From 2825f33ded98415fbd5bac31c6c74a3618b999a9 Mon Sep 17 00:00:00 2001 From: Cedric Date: Sun, 2 Oct 2022 19:15:59 -1000 Subject: [PATCH 64/94] conda env setup improvement --- docs/mana.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/mana.md b/docs/mana.md index 030749c..d3c3230 100644 --- a/docs/mana.md +++ b/docs/mana.md @@ -29,10 +29,10 @@ To setup nextflow on your account, follow these steps. ```bash # Load the latest anaconda3 module -module load lang/Anaconda3 +module load lang/Anaconda3/2022.05 # Initialize environment -. /opt/apps/software/lang/Anaconda3/5.1.0/etc/profile.d/conda.sh +. $(conda info --base)/etc/profile.d/conda.sh # Install nextflow (here in base environment, but you can create a new one if you'd like) conda install -c bioconda nextflow @@ -41,6 +41,6 @@ conda install -c bioconda nextflow If you want these settings to be persistent, you can add the first 2 commands in your .bash_profile file like this: ```bash -echo "module load lang/Anaconda3" >> ~/.bash_profile -echo ". /opt/apps/software/lang/Anaconda3/5.1.0/etc/profile.d/conda.sh >> ~/.bash_profile" -``` \ No newline at end of file +echo "module load lang/Anaconda3/2022.05" >> ~/.bash_profile +echo "$(conda info --base)/etc/profile.d/conda.sh" >> ~/.bash_profile +``` From 3f2ffa314d2e956eaf5c077aa3cc35d0fc25ad45 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 4 Oct 2022 00:25:16 +0100 Subject: [PATCH 65/94] Revised base and sarek configs to provide memory on per core basis --- conf/pipeline/sarek/sbc_sharc.config | 20 ++++++++++---------- conf/sbc_sharc.config | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index d970574..4ef0534 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -25,19 +25,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 6.h * task.attempt, 'time' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + memory = { check_max( 72.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 192.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +46,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 240.GB * task.attempt, 'memory' ) } } @@ -54,7 +54,7 @@ process { withName:'BWAMEM1_MEM|BWAMEM2_MEM' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 192.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -71,12 +71,12 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 72.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + memory = { check_max( 240.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 21dd18b..6583da7 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -42,7 +42,7 @@ process { executor = 'sge' penv = 'smp' queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } - clusterOptions = { "-l rmem=${task.memory.toGiga()}G" } + clusterOptions = { "-l rmem=${task.memory.toGiga()/task.cpus}G" } } From 584052cfff0713b3659cbd85b8826603c378128b Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 4 Oct 2022 09:12:29 +0100 Subject: [PATCH 66/94] Revised base and all pipeline configs to provide memory on per core basis --- conf/pipeline/atacseq/sbc_sharc.config | 20 +++++--------------- conf/pipeline/chipseq/sbc_sharc.config | 20 +++++--------------- conf/pipeline/rnaseq/sbc_sharc.config | 22 ++++++---------------- conf/pipeline/sarek/sbc_sharc.config | 16 +++------------- conf/sbc_sharc.config | 16 ++++------------ 5 files changed, 23 insertions(+), 71 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 0660591..7cd0e4a 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/atacseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/atacseq // process-specific resource requirements - reduced specification from those in atacseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 6fef3ba..0e66333 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/chipseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/chipseq // process-specific resource requirements - reduced specification from those in chipseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8033609..d9d9878 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/rnaseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/rnaseq // process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +36,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 160.GB * task.attempt, 'memory' ) } } } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 4ef0534..3a7f5b9 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/sarek - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/sarek // process-specific resource requirements - reduced specification from those in sarek/conf/base.config @@ -77,7 +67,7 @@ process { withName:'GATK4_MARKDUPLICATES' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 240.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + time = { check_max( 12.h * task.attempt, 'time' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 6583da7..91cf4e3 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -1,14 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Base Institutional Configuration - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Base Institutional Configuration // nf-core specific parameters displayed in header summary of each run @@ -42,7 +33,7 @@ process { executor = 'sge' penv = 'smp' queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } - clusterOptions = { "-l rmem=${task.memory.toGiga()/task.cpus}G" } + clusterOptions = { "-l rmem=${ (task.memory.toGiga() / task.cpus) }G" } } @@ -65,3 +56,4 @@ singularity { autoMounts = true } + From ce39096f32d94071a9fdd2aab6e93c3bfaecbe7a Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:44:19 +0200 Subject: [PATCH 67/94] Create tigem.config --- conf/tigem.config | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 conf/tigem.config diff --git a/conf/tigem.config b/conf/tigem.config new file mode 100644 index 0000000..1e5adea --- /dev/null +++ b/conf/tigem.config @@ -0,0 +1,31 @@ +params { + config_profile_description = 'Telethon Institute of Genetic and Medicine (TIGEM) provided by nf-core/configs.' + config_profile_contact = 'Giuseppe Martone (@giusmar)' + config_profile_url = 'https://github.com/giusmar' +} + +process.executor = 'slurm' +google.zone = 'europe-west1' + +singularity { + enabled = true + autoMounts = true + cacheDir = 'work/singularity' +} + +timeline { + enabled = true + file = "${params.outdir}/execution/timeline.html" +} +report { + enabled = true + file = "${params.outdir}/execution/report.html" +} +trace { + enabled = true + file = "${params.outdir}/execution/trace.txt" +} +dag { + enabled = true + file = "${params.outdir}/execution/pipeline.svg" +} From da0482a904fb3ec4d9d6f92a942db89f79ecc7a4 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:52:50 +0200 Subject: [PATCH 68/94] Create tigem.md --- docs/tigem.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docs/tigem.md diff --git a/docs/tigem.md b/docs/tigem.md new file mode 100644 index 0000000..6997e82 --- /dev/null +++ b/docs/tigem.md @@ -0,0 +1,7 @@ +# nf-core/configs: TIGEM configuration + +To use, run the pipeline with -profile tigem. This will download and launch the tigem.config which has been pre-configured with a setup suitable for the TIGEM personal biocluster. + +--- + +For this pipeline you need a docker container containing the tools needed for the specific pipeline. You can find the docker containers on giusmar docker hub profile. A docker image containing the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. From 2479c1e7e8af9c512d11ea5023d940eb71d18337 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:53:27 +0200 Subject: [PATCH 69/94] Update tigem.md --- docs/tigem.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tigem.md b/docs/tigem.md index 6997e82..31e42de 100644 --- a/docs/tigem.md +++ b/docs/tigem.md @@ -1,6 +1,6 @@ # nf-core/configs: TIGEM configuration -To use, run the pipeline with -profile tigem. This will download and launch the tigem.config which has been pre-configured with a setup suitable for the TIGEM personal biocluster. +To use, run the pipeline with `-profile tigem`. This will download and launch the tigem.config which has been pre-configured with a setup suitable for the TIGEM personal biocluster. --- From bde7502ed278e90a3c0b8071ace5f65b0b30d330 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:54:13 +0200 Subject: [PATCH 70/94] Update nfcore_custom.config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index aba5349..97b66f2 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -69,6 +69,7 @@ profiles { sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} sbc_sharc { includeConfig "${params.custom_config_base}/conf/sbc_sharc.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} + tigem { includeConfig "${params.custom_config_base}/conf/tigem.config"} uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } unibe_ibu { includeConfig "${params.custom_config_base}/conf/unibe_ibu.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } From a4244eb3462475bd90109f06a2cdb8cb03ee1600 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 11:06:26 +0200 Subject: [PATCH 71/94] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 838d288..da1d6f1 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,7 @@ Currently documentation is available for the following systems: - [SANGER](docs/sanger.md) - [SBC_SHARC](docs/sbc_sharc.md) - [SEG_GLOBE](docs/seg_globe.md) +- [TIGEM](docs/tigem.md) - [UCT_HPC](docs/uct_hpc.md) - [UNIBE_IBU](docs/unibe_ibu.md) - [UPPMAX](docs/uppmax.md) From 5758687a091d40271ccda8716e5a2be3743d0afc Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 11:07:36 +0200 Subject: [PATCH 72/94] Update main.yml --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4d68ca..61af682 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,6 +87,7 @@ jobs: - "sanger" - "sbc_sharc" - "seg_globe" + - "tigem" - "uct_hpc" - "unibe_ibu" - "uppmax" From e5b798e21f157dec8d6cc51a7b705cdc940c1022 Mon Sep 17 00:00:00 2001 From: Marc Hoeppner Date: Tue, 4 Oct 2022 14:07:44 +0200 Subject: [PATCH 73/94] Updating CCGA med to use the new Beegfs file system --- conf/ccga_med.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/ccga_med.config b/conf/ccga_med.config index d55fde2..c9b7b44 100644 --- a/conf/ccga_med.config +++ b/conf/ccga_med.config @@ -13,8 +13,8 @@ params { singularity { enabled = true - runOptions = "-B /work_ifs -B /scratch" - cacheDir = "/work_ifs/ikmb_repository/singularity_cache/" + runOptions = "-B /work_ifs -B /scratch -B /work_beegfs" + cacheDir = "/work_beegfs/ikmb_repository/singularity_cache/" } executor { @@ -31,7 +31,7 @@ process { params { // illumina iGenomes reference file paths on RZCluster - igenomes_base = '/work_ifs/ikmb_repository/references/iGenomes/references/' + igenomes_base = '/work_beegfs/ikmb_repository/references/iGenomes/references/' saveReference = true max_memory = 250.GB max_cpus = 24 From cade84aacd7d533f34c9e52d2abc39c54a290084 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 4 Oct 2022 19:50:51 +0000 Subject: [PATCH 74/94] [automated] Fix linting with Prettier --- docs/mana.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/mana.md b/docs/mana.md index d3c3230..4d0be0d 100644 --- a/docs/mana.md +++ b/docs/mana.md @@ -1,6 +1,6 @@ # nf-core/configs Mana (at University of Hawaii at Manoa) Configuration -To use, run the pipeline with `-profile mana`. It will use the following parameters for Mana (UHM HPCC): +To use, run the pipeline with `-profile mana`. It will use the following parameters for Mana (UHM HPCC): - Load singularity and use it as default container technology - Setup a container cache directory in your home (~/.singularity_images_cache) @@ -10,7 +10,6 @@ To use, run the pipeline with `-profile mana`. It will use the following paramet - Memory: 400.GB - Time: 72.h - ## Pre-requisites In order to run a nf-core pipeline on Mana, you will need to setup nextflow in your environment. From 34441a8f82a61b366991b1b15b24d5c65b8a29d7 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 4 Oct 2022 21:59:19 +0200 Subject: [PATCH 75/94] Update docs/gis.md --- docs/gis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gis.md b/docs/gis.md index 1864215..022605e 100644 --- a/docs/gis.md +++ b/docs/gis.md @@ -18,7 +18,7 @@ source /mnt/projects/rpd/rc/init.2017-04 module load miniconda3 -# Run a nextflow pipeline with dependencies bundles in a conda environment +# Run a nextflow pipeline with dependencies bundled in a conda environment set +u source activate nfcore-rnaseq-1.0dev set -u From baf857654c60d2fc42dde1601907022a21870047 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 22:28:38 +0200 Subject: [PATCH 76/94] Update tigem.md --- docs/tigem.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tigem.md b/docs/tigem.md index 31e42de..95d5415 100644 --- a/docs/tigem.md +++ b/docs/tigem.md @@ -4,4 +4,4 @@ To use, run the pipeline with `-profile tigem`. This will download and launch th --- -For this pipeline you need a docker container containing the tools needed for the specific pipeline. You can find the docker containers on giusmar docker hub profile. A docker image containing the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +This pipeline can be used on TIGEM clusters, in which is installed slurm as job scheduling system, so you can use this config if you have the same tool installed. An additional parameter is google.zone to allow downloading data from GCP for a specific time zone. It should not interfere with a local or an AWS configuration. From e37bccc9d8dff9beff40ae8512b48a0224cd6a00 Mon Sep 17 00:00:00 2001 From: Giuseppe Martone <46596039+giusmar@users.noreply.github.com> Date: Tue, 4 Oct 2022 22:58:50 +0200 Subject: [PATCH 77/94] Update tigem.config --- conf/tigem.config | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/conf/tigem.config b/conf/tigem.config index 1e5adea..b89a4ed 100644 --- a/conf/tigem.config +++ b/conf/tigem.config @@ -12,20 +12,3 @@ singularity { autoMounts = true cacheDir = 'work/singularity' } - -timeline { - enabled = true - file = "${params.outdir}/execution/timeline.html" -} -report { - enabled = true - file = "${params.outdir}/execution/report.html" -} -trace { - enabled = true - file = "${params.outdir}/execution/trace.txt" -} -dag { - enabled = true - file = "${params.outdir}/execution/pipeline.svg" -} From e2146b5905a7b2bf44da160c052c944a6d80055b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 5 Oct 2022 09:59:07 +0200 Subject: [PATCH 78/94] Update docs/tigem.md --- docs/tigem.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tigem.md b/docs/tigem.md index 95d5415..e562fe4 100644 --- a/docs/tigem.md +++ b/docs/tigem.md @@ -4,4 +4,4 @@ To use, run the pipeline with `-profile tigem`. This will download and launch th --- -This pipeline can be used on TIGEM clusters, in which is installed slurm as job scheduling system, so you can use this config if you have the same tool installed. An additional parameter is google.zone to allow downloading data from GCP for a specific time zone. It should not interfere with a local or an AWS configuration. +This configuration profile can be used on TIGEM clusters, with the pre-installed SLURM job scheduling system. An additional parameter is `google.zone` to allow downloading data from GCP for a specific time zone. It should not interfere with any local or other AWS configuration. From 0cabf67b6193f162bd907765021594cb6acbd396 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 4 Oct 2022 15:09:56 -0500 Subject: [PATCH 79/94] feat(demultiplex): Add optimized transfers for bases2fastq --- conf/pipeline/demultiplex/aws_tower.config | 29 ++++++++++++++++++++++ pipeline/demultiplex.config | 13 ++++++++++ 2 files changed, 42 insertions(+) create mode 100644 conf/pipeline/demultiplex/aws_tower.config create mode 100644 pipeline/demultiplex.config diff --git a/conf/pipeline/demultiplex/aws_tower.config b/conf/pipeline/demultiplex/aws_tower.config new file mode 100644 index 0000000..520487f --- /dev/null +++ b/conf/pipeline/demultiplex/aws_tower.config @@ -0,0 +1,29 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'Edmund Miller(@emiller88)' + config_profile_description = 'nf-core/demultiplex AWS Tower profile provided by nf-core/configs' +} + +aws { + batch { + maxParallelTransfers = 24 + maxTransferAttempts = 3 + } + client { + maxConnections = 24 + uploadMaxThreads = 24 + maxErrorRetry = 3 + socketTimeout = 3600000 + uploadRetrySleep = 1000 + uploadChunkSize = 32.MB + } +} + +process { + withName: BASES2FASTQ { + cpus = 16 + memory = 48.GB + } +} diff --git a/pipeline/demultiplex.config b/pipeline/demultiplex.config new file mode 100644 index 0000000..46c3e08 --- /dev/null +++ b/pipeline/demultiplex.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/demultiplex custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/demultiplex folder and imported + * under a profile name here. + */ + +profiles { + aws_tower { includeConfig "${params.custom_config_base}/conf/pipeline/demultiplex/aws_tower.config" } +} From 33708acdc58a91f38f62b0b1fe0f3e372e57bf2a Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Tue, 4 Oct 2022 15:22:26 -0500 Subject: [PATCH 80/94] docs(demultiplex): Add aws_tower docs --- README.md | 2 ++ docs/pipeline/demultiplex/aws_tower.md | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docs/pipeline/demultiplex/aws_tower.md diff --git a/README.md b/README.md index ddf9267..50c34aa 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,8 @@ Currently documentation is available for the following pipelines within specific - [SBC_SHARC](docs/pipeline/atacseq/sbc_sharc.md) - chipseq - [SBC_SHARC](docs/pipeline/chipseq/sbc_sharc.md) +- demultiplex + - [AWS_TOWER](docs/pipeline/demultiplex/aws_tower.md) - eager - [EVA](docs/pipeline/eager/eva.md) - mag diff --git a/docs/pipeline/demultiplex/aws_tower.md b/docs/pipeline/demultiplex/aws_tower.md new file mode 100644 index 0000000..eb87186 --- /dev/null +++ b/docs/pipeline/demultiplex/aws_tower.md @@ -0,0 +1,19 @@ +# nf-core/configs: AWS Tower Demultiplex specific configuration + +Extra specific configuration for demultiplex pipeline + +## Usage + +To use, run the pipeline with `-profile aws_tower`. + +This will download and launch the demultiplex specific [`aws_tower.config`](../../../conf/pipeline/demultiplex/aws_tower.config) which has been pre-configured with a setup suitable for AWS batch through tower. + +Example: `nextflow run nf-core/demultiplex -profile aws_tower` + +## eager specific configurations for eva + +Specific configurations for AWS has been made for demultiplex. + +### General profiles + +- The general AWS Tower profile runs with default nf-core/demultiplex parameters, but with modifications to account file transfer speed and optimized bases2fastq resources. From fb17e0c7f45b25e900e16993bd55357d8183b0f1 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 10 Oct 2022 14:30:56 +0200 Subject: [PATCH 81/94] Explain how to install nf-core through modules fixes #345 --- docs/uppmax.md | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 891762a..8baed08 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -8,7 +8,15 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ ## Using the UPPMAX config profile -Before running the pipeline you will need to either install `Nextflow` or load it using the environment module system (this can be done with e.g. `module load bioinfo-tools Nextflow/` where `VERSION` is e.g. `20.10`). +The recommended way to activate `Nextflow`, `nf-tools`, and any pipeline +available in `nf-core` on UPPMAX is to use the [module system](https://www.uppmax.uu.se/resources/software/module-system/): +```bash +# Log in to the desired cluster +ssh @{rackham,miarka,bianca}.uppmax.uu.se + +# Activate the modules, you can also choose to use a specific version with e.g. `Nextflow/21.10`. +module load bioinfo-tools Nextflow nf-core nf-core-pipelines +``` To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. @@ -94,20 +102,12 @@ Before running a nf-core pipeline on `bianca` you will first have to download th In this guide, we use `rackham` to download and transfer files to the `wharf` area, but it can also be done on your own computer. If you use `rackham` to download the pipeline and the singularity containers, we recommend using an interactive session (cf [interactive guide](https://www.uppmax.uu.se/support/faq/running-jobs-faq/how-can-i-run-interactively-on-a-compute-node/)), which is what we do in the following guide. -### Download and install Nextflow - -You can use the `Nextflow` UPPMAX provided `module`, but if necessary, you can also download a more recent version. - -```bash -# Connect to bianca -$ ssh -A -@bianca.uppmax.uu.se - -# See the available versions for the module -module spider Nextflow +It is recommended to activate `Nextflow`, `nf-tools` and your `nf-core` +pipeline through the module system (see **Using the UPPMAX config profile** +above). In case you need a specific version of any of these tools you can +follow the guide below. -# Load a specific version of the Nextflow module -module load bioinfo-tools Nextflow/` -``` +### Download and install Nextflow ```bash # Connect to rackham @@ -160,19 +160,6 @@ $ export NXF_SINGULARITY_CACHEDIR=/castor/project/proj_nobackup/singularity-imag ### Install nf-core tools -You can use the `nf-core` UPPMAX provided `module`, but if necessary, you can also download a more recent version. - -```bash -# Connect to rackham -$ ssh -X @rackham.uppmax.uu.se - -# See the available versions for the module -module spider nf-core - -# Load a specific version of the nf-core module -module load bioinfo-tools nf-core/` -``` - ```bash # Connect to rackham $ ssh -X @rackham.uppmax.uu.se @@ -254,7 +241,7 @@ And then `nf-core/` can be used with: $ nextflow run ~/ -profile uppmax --project --genome ... ``` -## Update a pipeline +### Update a pipeline To update, repeat the same steps as for installing and update the link. From f200f6ab44580ad688daa8760de99e78278c99a6 Mon Sep 17 00:00:00 2001 From: Praveen Raj S <43108054+praveenraj2018@users.noreply.github.com> Date: Mon, 10 Oct 2022 16:01:22 +0200 Subject: [PATCH 82/94] Disable mounting of KI lab folders Do not mount KI lab folders. The drives are not used anymore. --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 5f794d6..0fca214 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -29,7 +29,7 @@ process { singularity { cacheDir = '/data1/containers/' enabled = true - runOptions = "--bind /media/BTB_2021_01" + //runOptions = "--bind /media/BTB_2021_01" } // To use docker, use nextflow run -profile munin,docker From ec8d67285af1a71161a953b16bc8d78aea5fe68c Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 10 Oct 2022 16:03:39 +0200 Subject: [PATCH 83/94] Run prettier --- docs/uppmax.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/uppmax.md b/docs/uppmax.md index 8baed08..07b512c 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -10,6 +10,7 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ The recommended way to activate `Nextflow`, `nf-tools`, and any pipeline available in `nf-core` on UPPMAX is to use the [module system](https://www.uppmax.uu.se/resources/software/module-system/): + ```bash # Log in to the desired cluster ssh @{rackham,miarka,bianca}.uppmax.uu.se From 278ae391a0b69958abdd9d521524e3a9a775f5e8 Mon Sep 17 00:00:00 2001 From: Adrien Coulier Date: Mon, 10 Oct 2022 16:21:47 +0200 Subject: [PATCH 84/94] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit replace `nf-tools` by `nf-core` Co-authored-by: Matthias Hörtenhuber --- docs/uppmax.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 07b512c..dd664fb 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -8,7 +8,7 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ ## Using the UPPMAX config profile -The recommended way to activate `Nextflow`, `nf-tools`, and any pipeline +The recommended way to activate `Nextflow`, `nf-core`, and any pipeline available in `nf-core` on UPPMAX is to use the [module system](https://www.uppmax.uu.se/resources/software/module-system/): ```bash @@ -103,7 +103,7 @@ Before running a nf-core pipeline on `bianca` you will first have to download th In this guide, we use `rackham` to download and transfer files to the `wharf` area, but it can also be done on your own computer. If you use `rackham` to download the pipeline and the singularity containers, we recommend using an interactive session (cf [interactive guide](https://www.uppmax.uu.se/support/faq/running-jobs-faq/how-can-i-run-interactively-on-a-compute-node/)), which is what we do in the following guide. -It is recommended to activate `Nextflow`, `nf-tools` and your `nf-core` +It is recommended to activate `Nextflow`, `nf-core` and your `nf-core` pipeline through the module system (see **Using the UPPMAX config profile** above). In case you need a specific version of any of these tools you can follow the guide below. From adbf728ce522a1c69237c8c24457c1d69330a115 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:54:30 +0200 Subject: [PATCH 85/94] vsc_ugent: limit queuesize --- conf/vsc_ugent.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/vsc_ugent.config b/conf/vsc_ugent.config index 0bc6ffd..b46b347 100644 --- a/conf/vsc_ugent.config +++ b/conf/vsc_ugent.config @@ -7,9 +7,11 @@ workDir = "$scratch_dir/work" // Perform work directory cleanup when the run has succesfully completed // cleanup = true -// Reduce the job submit rate to about 5 per second, this way the server won't be bombarded with jobs +// Reduce the job submit rate to about 3 per second, this way the server won't be bombarded with jobs +// Limit queueSize to keep job rate under control and avoid timeouts executor { submitRateLimit = '3 sec' + queueSize = 50 } // Specify that singularity should be used and where the cache dir will be for the images @@ -114,4 +116,4 @@ profiles { scratch = "$scratch_dir" } } -} \ No newline at end of file +} From 9b971c2b7655e661251f5c981c3be14ba1a69177 Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Tue, 18 Oct 2022 14:03:07 +0000 Subject: [PATCH 86/94] Ucl_myriad --- .github/workflows/main.yml | 1 + README.md | 1 + conf/ucl_myriad.config | 34 +++++++++++++++++++++++++ docs/ucl_myriad.md | 52 ++++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 5 files changed, 89 insertions(+) create mode 100644 conf/ucl_myriad.config create mode 100644 docs/ucl_myriad.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55488be..3936468 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,6 +89,7 @@ jobs: - "sbc_sharc" - "seg_globe" - "tigem" + - "ucl_myriad" - "uct_hpc" - "unibe_ibu" - "uppmax" diff --git a/README.md b/README.md index 50c34aa..fe38eb9 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,7 @@ Currently documentation is available for the following systems: - [SBC_SHARC](docs/sbc_sharc.md) - [SEG_GLOBE](docs/seg_globe.md) - [TIGEM](docs/tigem.md) +- [UCL_MYRIAD](docs/ucl_myriad.md) - [UCT_HPC](docs/uct_hpc.md) - [UNIBE_IBU](docs/unibe_ibu.md) - [UPPMAX](docs/uppmax.md) diff --git a/conf/ucl_myriad.config b/conf/ucl_myriad.config new file mode 100644 index 0000000..3f9425c --- /dev/null +++ b/conf/ucl_myriad.config @@ -0,0 +1,34 @@ +params { + + config_profile_description = 'University College London Myriad cluster' + config_profile_contact = 'Chris Wyatt (ucbtcdr@ucl.ac.uk)' + config_profile_url = 'https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/' + +} + +process { + executor='sge' + penv = 'smp' +} + +params { + // Defaults only, expecting to be overwritten + max_memory = 128.GB + max_cpus = 36 + max_time = 72.h + // igenomes_base = 's3://ngi-igenomes/igenomes/' +} + +// optional executor settings + +executor { + + queueSize = 10 + submitRateLimit = '1 sec' + +} + +singularity { + enabled = true + autoMounts = true +} \ No newline at end of file diff --git a/docs/ucl_myriad.md b/docs/ucl_myriad.md new file mode 100644 index 0000000..4d32658 --- /dev/null +++ b/docs/ucl_myriad.md @@ -0,0 +1,52 @@ +# nf-core/configs: Myriad Configuration + +All nf-core pipelines have been successfully configured for use on UCL's myriad cluster [University College London](https://www.rc.ucl.ac.uk/docs/Clusters/Myriad/). + +To use, run the pipeline with `-profile ucl_myriad`. This will download and launch the [`ucl_myriad.config`](../conf/ucl_myriad.config) which has been pre-configured with a setup suitable for the myriad cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Using Nextflow on Myriad + +Before running the pipeline you will need to install and configure Nextflow and Singularity. + +### Singularity + +This can be done with the following commands: + +```bash +## Load Singularity environment modules - these commands can be placed in your ~/.bashrc also +module add java/openjdk-11/11.0.1 +module add singularity-env/1.0.0 + +``` + +Then set the correct configuration of the cache directories, where is replaced with you credentials which you can find by entering `whoami` into the terminal once you are logged into myriad. Once you have added your credentials save these lines into your .bashrc file in the base directory (e.g. /home//.bashrc): + +``` +# Set all the Singularity cache dirs to Scratch +export SINGULARITY_CACHEDIR=/home//Scratch/.singularity/ +export SINGULARITY_TMPDIR=/home//Scratch/.singularity/tmp +export SINGULARITY_LOCALCACHEDIR=/home//Scratch/.singularity/localcache +export SINGULARITY_PULLFOLDER=/home//Scratch/.singularity/pull + +# Bind your Scratch directory so it is accessible from inside the container +export SINGULARITY_BINDPATH=/scratch/scratch/ +``` + +### Nextflow + +Download the latest release of nextflow. Warning: the self-update line should update to the latest version, but sometimes not, so please check which is the latest release (https://github.com/nextflow-io/nextflow/releases), you can then manually set this by entering (`NXF_VER=XX.XX.X`). + +```bash +## Download Nextflow-all +curl -s https://get.nextflow.io | bash +NXF_VER=22.10.0 +nextflow -self-update +chmod a+x nextflow +mv nextflow ~/bin/nextflow +``` + +Then make sure that your bin PATH is executable, by placing the following line in your .bashrc: + +``` +export PATH=$PATH:/home//bin +``` \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 83390c4..51c1121 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -71,6 +71,7 @@ profiles { sbc_sharc { includeConfig "${params.custom_config_base}/conf/sbc_sharc.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} tigem { includeConfig "${params.custom_config_base}/conf/tigem.config"} + ucl_myriad { includeConfig "${params.custom_config_base}/conf/ucl_myriad.config"} uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } unibe_ibu { includeConfig "${params.custom_config_base}/conf/unibe_ibu.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } From f4c13e6511522311f101217d64ecec21f49dbe6a Mon Sep 17 00:00:00 2001 From: chriswyatt1 Date: Tue, 18 Oct 2022 14:10:55 +0000 Subject: [PATCH 87/94] fix_trainling_nl --- docs/ucl_myriad.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/ucl_myriad.md b/docs/ucl_myriad.md index 4d32658..1884a48 100644 --- a/docs/ucl_myriad.md +++ b/docs/ucl_myriad.md @@ -16,12 +16,11 @@ This can be done with the following commands: ## Load Singularity environment modules - these commands can be placed in your ~/.bashrc also module add java/openjdk-11/11.0.1 module add singularity-env/1.0.0 - ``` Then set the correct configuration of the cache directories, where is replaced with you credentials which you can find by entering `whoami` into the terminal once you are logged into myriad. Once you have added your credentials save these lines into your .bashrc file in the base directory (e.g. /home//.bashrc): -``` +```bash # Set all the Singularity cache dirs to Scratch export SINGULARITY_CACHEDIR=/home//Scratch/.singularity/ export SINGULARITY_TMPDIR=/home//Scratch/.singularity/tmp @@ -47,6 +46,6 @@ mv nextflow ~/bin/nextflow Then make sure that your bin PATH is executable, by placing the following line in your .bashrc: -``` +```bash export PATH=$PATH:/home//bin -``` \ No newline at end of file +``` From 7547fba92e52be5b295cfef81ad0b047fd3311fb Mon Sep 17 00:00:00 2001 From: Athanasios Baltzis Date: Thu, 20 Oct 2022 18:00:14 +0200 Subject: [PATCH 88/94] Add proteinfold-specific CRG config --- README.md | 2 ++ conf/pipeline/proteinfold/crg.config | 27 +++++++++++++++++++++++++++ docs/pipeline/proteinfold/crg.md | 23 +++++++++++++++++++++++ pipeline/proteinfold.config | 13 +++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 conf/pipeline/proteinfold/crg.config create mode 100644 docs/pipeline/proteinfold/crg.md create mode 100644 pipeline/proteinfold.config diff --git a/README.md b/README.md index fe38eb9..a066053 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,8 @@ Currently documentation is available for the following pipelines within specific - taxprofiler - [EVA](docs/pipeline/taxprofiler/eva.md) - [hasta](docs/pipeline/taxprofiler/hasta.md) +- proteinfold + - [CRG](docs/pipeline/proteinfold/crg.md) ### Pipeline-specific documentation diff --git a/conf/pipeline/proteinfold/crg.config b/conf/pipeline/proteinfold/crg.config new file mode 100644 index 0000000..f778d8c --- /dev/null +++ b/conf/pipeline/proteinfold/crg.config @@ -0,0 +1,27 @@ +profiles { + crg { + params { + config_profile_contact = 'Athanasios Baltzis (@athbaltzis)' + config_profile_description = 'nf-core/proteinfold CRG profile provided by nf-core/configs' + } + executor.name = 'crg' + process { + queue = 'short-sl7,long-sl7' + withName: 'RUN_AF2|RUN_AF2_PRED|COLABFOLD_BATCH' { + cpus = 1 + memory = "30 GB" + queue = params.use_gpu ? 'gpu' : 'long-sl7' + clusterOptions = { ( task.queue == 'gpu' ? '-l gpu=1' : '' ) } + } + withName: 'ARIA2' { + time = '12h' + } + withName: 'MMSEQS_COLABFOLDSEARCH' { + queue = 'mem_512' + memory = "100 GB" + cpus = 8 + time = '12h' + } + } + } +} diff --git a/docs/pipeline/proteinfold/crg.md b/docs/pipeline/proteinfold/crg.md new file mode 100644 index 0000000..9b11d2a --- /dev/null +++ b/docs/pipeline/proteinfold/crg.md @@ -0,0 +1,23 @@ +# nf-core/configs: CRG proteinfold specific configuration + +Extra specific configuration for proteinfold pipeline + +## Usage + +To use, run the pipeline with `-profile crg`. + +This will download and launch the proteinfold specific [`crg.config`](../../../conf/pipeline/proteinfold/crg.config) which has been pre-configured with a setup suitable for the sge cluster. + +Example: `nextflow run nf-core/proteinfold -profile crg` + +## proteinfold specific configurations for CRG + +Specific configurations for CRG has been made for proteinfold. + +### General profiles + + + +### Contextual profiles + + diff --git a/pipeline/proteinfold.config b/pipeline/proteinfold.config new file mode 100644 index 0000000..e4f59b4 --- /dev/null +++ b/pipeline/proteinfold.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/proteinfold custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/proteinfold folder and imported + * under a profile name here. + */ + +profiles { + crg { includeConfig "${params.custom_config_base}/conf/pipeline/proteinfold/crg.config" } +} From d2e82e4fd2962295682bcc378d59454a823562c2 Mon Sep 17 00:00:00 2001 From: Athanasios Baltzis Date: Thu, 20 Oct 2022 18:15:51 +0200 Subject: [PATCH 89/94] Add profile to main.yml --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3936468..72f11e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,6 +98,7 @@ jobs: - "uzh" - "vai" - "vsc_ugent" + - "crg" steps: - uses: actions/checkout@v1 - name: Install Nextflow From 002fc843624ad475dc3fe8f1c3e53cd1883bf260 Mon Sep 17 00:00:00 2001 From: Athanasios Baltzis Date: Thu, 20 Oct 2022 18:41:40 +0200 Subject: [PATCH 90/94] Add crg custom config file and update nfcore_custom.config --- conf/crg.config | 14 ++++++++++++++ nfcore_custom.config | 1 + 2 files changed, 15 insertions(+) create mode 100755 conf/crg.config diff --git a/conf/crg.config b/conf/crg.config new file mode 100755 index 0000000..7216b80 --- /dev/null +++ b/conf/crg.config @@ -0,0 +1,14 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'Centre for Genomic Regulation (CRG) cluster profile provided by nf-core/configs' + config_profile_contact = 'Athanasios Baltzis (@athbaltzis)' + config_profile_url = 'http://www.linux.crg.es/index.php/Main_Page' +} + +process { + executor = 'crg' +} + +singularity { + enabled = true +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 51c1121..aa62e0f 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,6 +30,7 @@ profiles { cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } cheaha { includeConfig "${params.custom_config_base}/conf/cheaha.config" } computerome { includeConfig "${params.custom_config_base}/conf/computerome.config" } + crg { includeConfig "${params.custom_config_base}/conf/crg.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } crukmi { includeConfig "${params.custom_config_base}/conf/crukmi.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } From 0a333e29b4c03b497c5e4107b9c4d2666c5635be Mon Sep 17 00:00:00 2001 From: Athanasios Baltzis Date: Fri, 21 Oct 2022 14:02:12 +0200 Subject: [PATCH 91/94] Update nfcore_custom.config Co-authored-by: Jose Espinosa-Carrasco --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index aa62e0f..53d7682 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,7 +30,7 @@ profiles { cfc_dev { includeConfig "${params.custom_config_base}/conf/cfc_dev.config" } cheaha { includeConfig "${params.custom_config_base}/conf/cheaha.config" } computerome { includeConfig "${params.custom_config_base}/conf/computerome.config" } - crg { includeConfig "${params.custom_config_base}/conf/crg.config" } + crg { includeConfig "${params.custom_config_base}/conf/crg.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } crukmi { includeConfig "${params.custom_config_base}/conf/crukmi.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } From 04bf413415b2b20d7a179f91dd1f1986476a10c9 Mon Sep 17 00:00:00 2001 From: Athanasios Baltzis Date: Fri, 21 Oct 2022 14:03:40 +0200 Subject: [PATCH 92/94] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 72f11e7..66bf36b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,6 +49,7 @@ jobs: - "cfc_dev" - "cheaha" - "computerome" + - "crg" - "crick" - "crukmi" - "denbi_qbic" @@ -98,7 +99,6 @@ jobs: - "uzh" - "vai" - "vsc_ugent" - - "crg" steps: - uses: actions/checkout@v1 - name: Install Nextflow From 81c5857c516c80ca1ab31a0d97aea9a9a95b1077 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 26 Oct 2022 11:07:14 +0200 Subject: [PATCH 93/94] Update vsc_ugent.config --- conf/vsc_ugent.config | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/conf/vsc_ugent.config b/conf/vsc_ugent.config index b46b347..d2ce550 100644 --- a/conf/vsc_ugent.config +++ b/conf/vsc_ugent.config @@ -10,10 +10,16 @@ workDir = "$scratch_dir/work" // Reduce the job submit rate to about 3 per second, this way the server won't be bombarded with jobs // Limit queueSize to keep job rate under control and avoid timeouts executor { - submitRateLimit = '3 sec' + submitRateLimit = '30/1min' queueSize = 50 } +// Add backoff strategy to catch cluster timeouts +process { + errorStrategy = { sleep(Math.pow(2, task.attempt) * 200 as long); return 'retry' } + maxRetries = 5 +} + // Specify that singularity should be used and where the cache dir will be for the images singularity { enabled = true @@ -30,7 +36,7 @@ profiles { skitty { params { config_profile_description = 'HPC_SKITTY profile for use on the Skitty cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 177.GB max_cpus = 36 @@ -40,7 +46,6 @@ profiles { process { executor = 'slurm' queue = 'skitty' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -48,7 +53,7 @@ profiles { swalot { params { config_profile_description = 'HPC_SWALOT profile for use on the Swalot cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 116.GB max_cpus = 20 @@ -58,7 +63,6 @@ profiles { process { executor = 'slurm' queue = 'swalot' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -66,7 +70,7 @@ profiles { victini { params { config_profile_description = 'HPC_VICTINI profile for use on the Victini cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 88.GB max_cpus = 36 @@ -76,7 +80,6 @@ profiles { process { executor = 'slurm' queue = 'victini' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -84,7 +87,7 @@ profiles { kirlia { params { config_profile_description = 'HPC_KIRLIA profile for use on the Kirlia cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 738.GB max_cpus = 36 @@ -94,7 +97,6 @@ profiles { process { executor = 'slurm' queue = 'kirlia' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -102,7 +104,7 @@ profiles { doduo { params { config_profile_description = 'HPC_DODUO profile for use on the Doduo cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 250.GB max_cpus = 96 @@ -112,7 +114,6 @@ profiles { process { executor = 'slurm' queue = 'doduo' - maxRetries = 2 scratch = "$scratch_dir" } } From ec190ac0136b0a796b8a799ee07177a5400561a2 Mon Sep 17 00:00:00 2001 From: Matthias De Smet <11850640+matthdsm@users.noreply.github.com> Date: Wed, 26 Oct 2022 11:09:45 +0200 Subject: [PATCH 94/94] Update conf/vsc_ugent.config Co-authored-by: nvnieuwk <101190534+nvnieuwk@users.noreply.github.com> --- conf/vsc_ugent.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/vsc_ugent.config b/conf/vsc_ugent.config index d2ce550..4a3733a 100644 --- a/conf/vsc_ugent.config +++ b/conf/vsc_ugent.config @@ -7,7 +7,7 @@ workDir = "$scratch_dir/work" // Perform work directory cleanup when the run has succesfully completed // cleanup = true -// Reduce the job submit rate to about 3 per second, this way the server won't be bombarded with jobs +// Reduce the job submit rate to about 30 per minute, this way the server won't be bombarded with jobs // Limit queueSize to keep job rate under control and avoid timeouts executor { submitRateLimit = '30/1min'