From 885f0ce5f69abafac3237728c929ba5b952eb593 Mon Sep 17 00:00:00 2001 From: Gisela Gabernet Garriga Date: Mon, 9 Nov 2020 22:41:38 +0100 Subject: [PATCH 01/86] awsbatch config update --- conf/awsbatch.config | 14 +++++++++++++- docs/awsbatch.md | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 2e5f83c..43f90db 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -7,7 +7,19 @@ params { awsqueue = false awsregion = 'eu-west-1' awscli = '/home/ec2-user/miniconda/bin/aws' - tracedir = './' +} + +timeline { + overwrite = true +} +report { + overwrite = true +} +trace { + overwrite = true +} +dag { + overwrite = true } process.executor = 'awsbatch' diff --git a/docs/awsbatch.md b/docs/awsbatch.md index c00acef..daf425b 100644 --- a/docs/awsbatch.md +++ b/docs/awsbatch.md @@ -1,4 +1,6 @@ # nf-core/configs: awsbatch Configuration To be used with `awsbatch`. -Custom queue, region and CLI path can be supplied with `params.awsqueue`, `params.awsregion`, `params.awscli`, respectively. +Custom queue and region can be supplied with `params.awsqueue`, `params.awsregion`, `params.awscli`, respectively. + +Allow `overwrite` of `trace`, `timeline`, `report` and `dag` to allow resuming pipelines. From 9b5bcbd43436fd086d53790c73202343dbbc58bd Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 9 Feb 2021 22:06:59 +0100 Subject: [PATCH 02/86] update aws clipath --- conf/awsbatch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 43f90db..a8b61b8 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -25,4 +25,4 @@ dag { process.executor = 'awsbatch' process.queue = params.awsqueue aws.region = params.awsregion -executor.awscli = params.awscli +aws.batch.cliPath = params.awscli From 537f52a6409200ac843e457722f0a74660d7d28c Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Feb 2021 17:50:20 +0100 Subject: [PATCH 03/86] Add MPI-EVA profile --- README.md | 30 ++--- conf/eva.config | 52 ++++++++ conf/pipeline/eager/eva.config | 212 +++++++++++++++++++++++++++++++++ docs/eva.md | 29 +++++ docs/pipeline/eager/eva.md | 34 ++++++ nfcore_custom.config | 1 + 6 files changed, 344 insertions(+), 14 deletions(-) create mode 100644 conf/eva.config create mode 100644 conf/pipeline/eager/eva.config create mode 100644 docs/eva.md create mode 100644 docs/pipeline/eager/eva.md diff --git a/README.md b/README.md index 45721e8..56c73b6 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,20 @@ A repository for hosting Nextflow configuration files containing custom paramete ## Table of contents -* [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) - * [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) - * [Pipeline-specific documentation](#pipeline-specific-documentation) - * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) - * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) -* [Help](#help) +- [Using an existing config](#using-an-existing-config) + - [Configuration and parameters](#configuration-and-parameters) + - [Offline usage](#offline-usage) +- [Adding a new config](#adding-a-new-config) + - [Checking user hostnames](#checking-user-hostnames) + - [Testing](#testing) + - [Documentation](#documentation) + - [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +- [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + - [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) + - [Pipeline-specific documentation](#pipeline-specific-documentation) + - [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) + - [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) +- [Help](#help) ## Using an existing config @@ -107,6 +107,7 @@ Currently documentation is available for the following systems: * [CZBIOHUB_AWS](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) * [EBC](docs/ebc.md) +* [EVA](docs/eva.md) * [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) @@ -174,6 +175,7 @@ Currently documentation is available for the following pipelines within specific * [UPPMAX](docs/pipeline/ampliseq/uppmax.md) * eager * [SHH](docs/pipeline/eager/shh.md) + * [EVA](docs/pipeline/eager/eva.md) * rnafusion * [MUNIN](docs/pipeline/rnafusion/munin.md) * sarek diff --git a/conf/eva.config b/conf/eva.config new file mode 100644 index 0000000..497085d --- /dev/null +++ b/conf/eva.config @@ -0,0 +1,52 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'Generic MPI-EVA cluster(s) profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://eva.mpg.de' +} + +// Preform work directory cleanup after a successful run +cleanup = true + +singularity { + enabled = true + autoMounts = true +} + +process { + executor = 'sge' + penv = 'smp' + queue = 'all.q' +} + +executor { + queueSize = 8 +} + +profiles { + archgen { + params { + igenomes_base = "/projects1/public_data/igenomes/" + config_profile_description = 'MPI-EVA archgen profile, provided by nf-core/configs.' + max_memory = 256.GB + max_cpus = 32 + max_time = 720.h + //Illumina iGenomes reference file path + igenomes_base = "/projects1/public_data/igenomes/" + } + + process { + queue = 'archgen.q' + } + + singularity { + cacheDir = "/mnt/archgen/users/singularity_scratch" + + } + + } + // Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option. + debug { + cleanup = false + } +} diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config new file mode 100644 index 0000000..8850cf1 --- /dev/null +++ b/conf/pipeline/eager/eva.config @@ -0,0 +1,212 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs' +} + +// Specific nf-core/eager process configuration +process { + + maxRetries = 2 + + // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion + clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toMega().toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().toString().replaceAll(/[\sB]/,'')}M" } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use + + withName: makeSeqDict { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: fastqc { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: adapter_removal { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: dedup { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: markduplicates { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: malt { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: maltextract { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: multivcfanalyzer { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: mtnucratio { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: vcf2genome { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: qualimap { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: damageprofiler { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: circularmapper { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: circulargenerator { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + + withName: preseq { + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + } + +} + +profiles { + + big_data { + + params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/eager big-data EVA profile provided by nf-core/configs' + } + + executor { + queueSize = 6 + } + + process { + + maxRetries = 2 + + withName:hostremoval_input_fastq { + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = 1440.h + } + + withLabel:'sc_tiny'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 2.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_small'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'sc_medium'{ + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_small'{ + cpus = { check_max( 2, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_medium' { + cpus = { check_max( 4, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_large'{ + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + + withLabel:'mc_huge'{ + cpus = { check_max( 32, 'cpus' ) } + memory = { check_max( 512.GB * task.attempt, 'memory' ) } + time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } + } + } + } + + pathogen_loose { + params { + config_profile_description = 'Pathogen (loose) MPI-EVA profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16 + } + } + pathogen_strict { + params { + config_profile_description = 'Pathogen (strict) MPI-EVA SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.1 + bwaalnl = 32 + } + } + human { + params { + config_profile_description = 'Human MPI-EVA SDAG profile, provided by nf-core/configs.' + bwaalnn = 0.01 + bwaalnl = 16500 + } + } +} diff --git a/docs/eva.md b/docs/eva.md new file mode 100644 index 0000000..d69ef03 --- /dev/null +++ b/docs/eva.md @@ -0,0 +1,29 @@ +# nf-core/configs: EVA Configuration + +All nf-core pipelines have been successfully configured for use on the Department of Genetics and Archaeogenetic's clusters at the [Max Planck Institute for Evolutionary Anthropology (MPI-EVA)](http://eva.mpg.de). + +To use, run the pipeline with `-profile eva`. You can further with optimise submissions by specifying which cluster queue you are using e,g, `-profile eva,archgen`. This will download and launch the [`eva.config`](../conf/eva.config) which has been pre-configured with a setup suitable for the `all.q` queue. The number of parallel jobs that run is currently limited to 8. + +Using this profile, a docker image containing all of the required software will be downloaded, and converted to a `singularity` image before execution of the pipeline. The image will currently be centrally stored here: + +## Additional Profiles + +We currently also offer profiles for the different department's specific nodes. + +### archgen + +If you specify `-profile eva,archgen` you will be able to use the nodes available on the `archgen.q` queue. + +Note the following characteristics of this profile: + +- By default, job resources are assigned a maximum number of CPUs of 32, 256 GB maximum memory and 720.h maximum wall time. +- Using this profile will currently store singularity images in a cache under `/mnt/archgen/users/singularity_scratch/cache/`. All archgen users currently have read/write access to this directory, however this will likely change to a read-only directory in the future that will be managed by the IT team. +- Intermediate files will be _automatically_ cleaned up (see `debug` below if you don't want this to happen) on successful run completion. + +>NB: You will need an account and VPN access to use the cluster at MPI-EVA in order to run the pipeline. If in doubt contact the IT team. +>NB: Nextflow will need to submit the jobs via SGE to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. + +### debug + +This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen` + diff --git a/docs/pipeline/eager/eva.md b/docs/pipeline/eager/eva.md new file mode 100644 index 0000000..4537182 --- /dev/null +++ b/docs/pipeline/eager/eva.md @@ -0,0 +1,34 @@ +# nf-core/configs: eva eager specific configuration + +Extra specific configuration for eager pipeline + +## Usage + +To use, run the pipeline with `-profile eva`. + +This will download and launch the eager specific [`eva.config`](../../../conf/pipeline/eager/eva.config) which has been pre-configured with a setup suitable for the MPI-EVA cluster. + +Example: `nextflow run nf-core/eager -profile eva` + +## eager specific configurations for eva + +Specific configurations for eva has been made for eager. + +### General profiles + +- The general MPI-EVA profile runs with default nf-core/eager parameters, but with modifications to account for issues SGE have with Java tools. + +#### big_data + +- This defines larger base computing resources for when working with very deep sequenced or high-endogenous samples. + +### Contextual profiles + +#### Human Pop-Gen + +* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) + +#### Pathogen + +* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) +* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 429e3c1..9e94f83 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,6 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } + eva { includeConfig "${params.custom_config_base}/conf/eva.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From a78aac327dde8ca0cdb84a011ccfc96c3e3a509e Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Feb 2021 17:58:50 +0100 Subject: [PATCH 04/86] Linting fixes --- docs/eva.md | 3 +-- docs/pipeline/eager/eva.md | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/eva.md b/docs/eva.md index d69ef03..42575b2 100644 --- a/docs/eva.md +++ b/docs/eva.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the Department of Genetics and Archaeogenetic's clusters at the [Max Planck Institute for Evolutionary Anthropology (MPI-EVA)](http://eva.mpg.de). -To use, run the pipeline with `-profile eva`. You can further with optimise submissions by specifying which cluster queue you are using e,g, `-profile eva,archgen`. This will download and launch the [`eva.config`](../conf/eva.config) which has been pre-configured with a setup suitable for the `all.q` queue. The number of parallel jobs that run is currently limited to 8. +To use, run the pipeline with `-profile eva`. You can further with optimise submissions by specifying which cluster queue you are using e,g, `-profile eva,archgen`. This will download and launch the [`eva.config`](../conf/eva.config) which has been pre-configured with a setup suitable for the `all.q` queue. The number of parallel jobs that run is currently limited to 8. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a `singularity` image before execution of the pipeline. The image will currently be centrally stored here: @@ -26,4 +26,3 @@ Note the following characteristics of this profile: ### debug This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen` - diff --git a/docs/pipeline/eager/eva.md b/docs/pipeline/eager/eva.md index 4537182..a8dc563 100644 --- a/docs/pipeline/eager/eva.md +++ b/docs/pipeline/eager/eva.md @@ -26,9 +26,9 @@ Specific configurations for eva has been made for eager. #### Human Pop-Gen -* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) +- `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`) #### Pathogen -* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) -* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) \ No newline at end of file +- `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) +- `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) From 5bfb9510c7e38ba4e43885abbd80b198a1bdfa83 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Sun, 14 Feb 2021 17:59:55 +0100 Subject: [PATCH 05/86] Add to testing --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e5c930..c500dfb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 4339e5872c211fe2186b8e6b0e012ace45749bb3 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 14 Feb 2021 18:05:06 +0100 Subject: [PATCH 06/86] Update eager.config --- pipeline/eager.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/eager.config b/pipeline/eager.config index 9242ecb..2827496 100644 --- a/pipeline/eager.config +++ b/pipeline/eager.config @@ -11,5 +11,5 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/pipeline/eager/shh.config" } mpcdf { includeConfig "${params.custom_config_base}/conf/pipeline/eager/mpcdf.config" } - + eva { includeConfig "${params.custom_config_base}/conf/pipeline/eager/eva.config" } } From 0bac3691e3656767aa10134c76ad4536e0b70814 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 14 Feb 2021 18:07:40 +0100 Subject: [PATCH 07/86] Create mpcdf.config --- docs/pipeline/eager/mpcdf.config | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 docs/pipeline/eager/mpcdf.config diff --git a/docs/pipeline/eager/mpcdf.config b/docs/pipeline/eager/mpcdf.config new file mode 100644 index 0000000..d170f4c --- /dev/null +++ b/docs/pipeline/eager/mpcdf.config @@ -0,0 +1,11 @@ +# nf-core/configs: mpcdf eager specific configuration + +Extra specific configuration for eager pipeline for the `cobra` cluster of the MPCDF + +## Usage + +To use, run the pipeline with `-profile mpcdf,cobra`. + +This will download and launch the eager specific [`mpcdf.config`](../../../conf/pipeline/eager/mpcdf.config) which has been pre-configured with a setup suitable for the mpcdf cluster. + +Currently this only applies to the `cobra` cluster, where maximum resources are adjusted accordingly. From 11e2083fb1ea0d38988ed6d68bf1b6051450237c Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 3 Mar 2021 09:57:34 +0000 Subject: [PATCH 08/86] Added eddie profile to nfcore_custom.config list --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 429e3c1..7abbb7e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,6 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } + eddie { includeConfig "${params.custom_config_bsae}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 3af2c6409f66f387c2b5aa7f039f435e0c74404c Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 3 Mar 2021 09:59:16 +0000 Subject: [PATCH 09/86] Initial commit of eddie.config --- conf/eddie.config | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 conf/eddie.config diff --git a/conf/eddie.config b/conf/eddie.config new file mode 100644 index 0000000..11ae1d9 --- /dev/null +++ b/conf/eddie.config @@ -0,0 +1,30 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'University of Edinburgh (eddie) cluster profile provided by nf-core/configs.' + config_profile_contact = 'Alison Meynert (@ameynert)' + config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' +} + +conda.createTimeout = '2h' + +process { + executor = 'sge' + + # memory environment & options + clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} + penv = 'sharedmem' + + # common SGE error statuses + errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} + maxErrors = '-1' + maxRetries = 3 + + # load module script for Anaconda + beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } + module = 'anaconda/5.3.1' +} +params { + saveReference = true + // illumina iGenomes reference file paths on GIS Aquila + igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' +} From 6f09a233f072e5878b5b8df803484d3ee3778f51 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 4 Mar 2021 14:46:23 +0000 Subject: [PATCH 10/86] Added singularity module --- conf/eddie.config | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 11ae1d9..5b156be 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -5,7 +5,13 @@ params { config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' } -conda.createTimeout = '2h' +conda { + createTimeout = '2h' +} + +singularity { + enabled = true +} process { executor = 'sge' @@ -19,12 +25,19 @@ process { maxErrors = '-1' maxRetries = 3 - # load module script for Anaconda + # load module script for Anaconda and Singularity beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } module = 'anaconda/5.3.1' + module = 'singularity/3.5.3' } + params { saveReference = true - // illumina iGenomes reference file paths on GIS Aquila + + // iGenomes reference base igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' + + max_memory = 384.GB + max_cpus = 32 + max_time = 240.h } From 4c0111f598935e7223c8d51f510a9271e959e41c Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 4 Mar 2021 14:46:58 +0000 Subject: [PATCH 11/86] Eddie profile documentation initial commit --- docs/eddie.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/eddie.md diff --git a/docs/eddie.md b/docs/eddie.md new file mode 100644 index 0000000..e6a2f87 --- /dev/null +++ b/docs/eddie.md @@ -0,0 +1,49 @@ +# nf-core/configs: Eddie Configuration + +nf-core pipelines sarek, rnaseq, and atacseq have all been tested on the University of Edinburgh Eddie HPC. + +## Getting help + +There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https://igmm.slack.com/channels/eddie3](https://igmm.slack.com/channels/eddie3) + +## Using the Eddie config profile + +To use, run the pipeline with `-profile eddie` (one hyphen). +This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). + +The configuration file supports running nf-core pipelines with either a Conda environment or Docker containers running under Singularity. + +```bash +nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +``` + +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. + +To list versions: + +```bash +module avail igmm/apps/nextflow +``` + +To load the most recent version: +```bash +module load igmm/apps/nextflow +``` + + +This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. + +To set up Nextflow on a login node ... TODO + +## Using iGenomes references + +A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. +You can do this by simply using the `--genome ` parameter. + +## Adjusting maximum resources + +This config is set for IGMM standard nodes which have 32 cores and 384GB memory. If you are a non-IGMM user, please see the [ECDF specification](https://www.wiki.ed.ac.uk/display/ResearchServices/Memory+Specification) and adjust the `--clusterOptions` flag appropriately, e.g. + +```bash +--clusterOptions "-C mem256GB" --max_memory "256GB" +``` From babeccd020011e62671b2df27c201a2cdb5cedae Mon Sep 17 00:00:00 2001 From: Graeme Grimes Date: Thu, 4 Mar 2021 16:45:21 +0000 Subject: [PATCH 12/86] changed # to // for comments --- conf/eddie.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 5b156be..3968783 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -16,16 +16,16 @@ singularity { process { executor = 'sge' - # memory environment & options + // memory environment & options clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} penv = 'sharedmem' - # common SGE error statuses + // common SGE error statuses errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} maxErrors = '-1' maxRetries = 3 - # load module script for Anaconda and Singularity + // load module script for Anaconda and Singularity beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } module = 'anaconda/5.3.1' module = 'singularity/3.5.3' From da3bd2693daf31531d0a57b6e046ea7b6e4a1d46 Mon Sep 17 00:00:00 2001 From: phue Date: Wed, 10 Mar 2021 11:26:01 +0100 Subject: [PATCH 13/86] cbe: add rapid qos and update config url --- conf/cbe.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 0a5763f..18f72dc 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -2,13 +2,13 @@ params { config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs' config_profile_contact = 'Patrick Hüther (@phue)' - config_profile_url = 'http://www.gmi.oeaw.ac.at/' + config_profile_url = 'https://clip.science' } process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } - clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } + clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } module = 'anaconda3/2019.10' } From e7f8189dfc95b07e34f8ea33ac51cc17d7a79d10 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 13:52:11 -0500 Subject: [PATCH 14/86] Create wcm.config --- conf/wcm.config | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 conf/wcm.config diff --git a/conf/wcm.config b/conf/wcm.config new file mode 100644 index 0000000..38cd3d1 --- /dev/null +++ b/conf/wcm.config @@ -0,0 +1,28 @@ +singularityDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow" + +params { + config_profile_description = 'Weill Cornell Medicine, Scientific Computing Unit Slurm cluster profile provided by nf-core/configs' + config_profile_contact = 'Ashley Stephen Doane, PhD (@DoaneAS)' + igenomes_base = '/athena/elementolab/scratch/reference/igenomes' +} + +singularity { + enabled = true + envWhitelist='SINGULARITY_BINDPATH' + cacheDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow" + autoMounts = true +} + +process { + executor = 'slurm' + queue = 'panda_physbio' + scratch = true + scratch = '/scratchLocal/`whoami`_${SLURM_JOBID}' +} + +params { + max_memory = 32.GB + max_cpus = 8 + max_time = 24.h +} + From d630bcfb6c12f92a54907de7b821a60a6f218a97 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 13:59:55 -0500 Subject: [PATCH 15/86] Create wcm.md --- docs/wcm.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/wcm.md diff --git a/docs/wcm.md b/docs/wcm.md new file mode 100644 index 0000000..74918ac --- /dev/null +++ b/docs/wcm.md @@ -0,0 +1,24 @@ +# nf-core/configs: Weill Cornell Medicine Configuration + +All nf-core pipelines have been successfully configured for use on the panda cluster at the WCM. + +To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Running the workflow on the Pasteur cluster + +Nextflow is not installed by default on the WCM cluster. + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +Nextflow manages each process as a separate job that is submitted to the cluster by using the `sbatch` command. +Nextflow shouldn't run directly on a login node but on a compute node or lab-specific interactive server when configured as a submit host. + +1. Run nextflow on a compute node or interactive server with submit host capability: + +```bash +# Run nextflow workflow +nextflow run \\ +nf-core/chipseq \\ +-resume \\ +-profile test,wcm +``` From 6343a7f87cbeaa24a81aa588998efefd0099ca57 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 14:01:07 -0500 Subject: [PATCH 16/86] Update wcm.md --- docs/wcm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wcm.md b/docs/wcm.md index 74918ac..0785829 100644 --- a/docs/wcm.md +++ b/docs/wcm.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the panda cluster at the WCM. -To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the WCM slurm cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. ## Running the workflow on the Pasteur cluster From cfe13fe6c1b72bbfeb27d56843c93f7714cc505d Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Mon, 15 Mar 2021 15:42:30 +0000 Subject: [PATCH 17/86] Fixed typo & added singularity tmpdir --- conf/eddie.config | 44 +++++++++++++++++++++++++------------------- nfcore_custom.config | 2 +- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 3968783..6145908 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -5,39 +5,45 @@ params { config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' } -conda { - createTimeout = '2h' -} - -singularity { - enabled = true +executor { + name = "sge" + queueSize = "100" } process { - executor = 'sge' - - // memory environment & options - clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} - penv = 'sharedmem' + clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus}" : null } + scratch = true + penv = { task.cpus > 1 ? "sharedmem" : null } // common SGE error statuses - errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} + errorStrategy = {task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish'} maxErrors = '-1' maxRetries = 3 - // load module script for Anaconda and Singularity - beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } - module = 'anaconda/5.3.1' - module = 'singularity/3.5.3' + beforeScript = + """ + '. /etc/profile.d/modules.sh + sleep 2 + module load 'anaconda/5.3.1' + module load 'roslin/singularity/3.5.3' + export SINGULARITY_TMPDIR="\$TMPDIR" + """ } params { saveReference = true - // iGenomes reference base - igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' - + igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes' max_memory = 384.GB max_cpus = 32 max_time = 240.h } + +env { + MALLOC_ARENA_MAX=1 +} + +singularity { + envWhitelist = "SINGULARITY_TMPDIR" + runOptions = '-p' +} \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 7abbb7e..b338f1c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_bsae}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 7e9f83ee8133993c4d14b1c1562644427e51655b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 15:54:19 +0100 Subject: [PATCH 18/86] Add IFB config --- .github/workflows/main.yml | 2 +- conf/ifb_core.config | 24 +++++++++++++++++++++++ docs/ifb_core.md | 40 ++++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 conf/ifb_core.config create mode 100644 docs/ifb_core.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e5c930..cf00bea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/conf/ifb_core.config b/conf/ifb_core.config new file mode 100644 index 0000000..50a72f4 --- /dev/null +++ b/conf/ifb_core.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The IFB core cluster profile' + config_profile_contact = 'https://community.france-bioinformatique.fr' + config_profile_url = 'https://www.france-bioinformatique.fr/' +} + +singularity { + // need one image per execution + enabled = true + runOptions = '-B /shared' +} + +process { + executor = 'slurm' +} + +params { + igenomes_ignore = true + // Max resources requested by a normal node on genotoul. + max_memory = 240.GB + max_cpus = 28 + max_time = 96.h +} diff --git a/docs/ifb_core.md b/docs/ifb_core.md new file mode 100644 index 0000000..0434a09 --- /dev/null +++ b/docs/ifb_core.md @@ -0,0 +1,40 @@ +# nf-core/configs: IFB core Configuration + +All nf-core pipelines have been successfully configured for use on the IFB core at the insert institution here. + +To use, run the pipeline with `-profile ifb_core`. This will download and launch the [`ifb_core.config`](../conf/ifb_core.config) which has been pre-configured with a setup suitable for the IFB core cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## How to use on IFB core + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on IFB core. You can do this by issuing the commands below: + +```bash +# Login to a compute node +srun --pty bash + +## Load Nextflow and Singularity environment modules +module purge +module load nextflow/20.04.1 + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +nf-core/workflow \\ +-resume +-profile ifb_core \\ +--email my-email@example.org \\ +-c my-specific.config +... + + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... + +``` + +## Databanks + +A local copy of several genomes are available in `/shared/bank` directory. See +our [databank page](https://ifb-elixirfr.gitlab.io/cluster/doc/banks/) +to search for your favorite genome. + +>NB: You will need an account to use the HPC cluster on IFB core in order to run the pipeline. If in doubt contact IT or go to [account page](https://my.cluster.france-bioinformatique.fr/manager2/login). diff --git a/nfcore_custom.config b/nfcore_custom.config index 429e3c1..a4c9939 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,6 +24,7 @@ profiles { czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } + ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } From cc17045914a56ec05d06da080bc536d20588631a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 15:58:29 +0100 Subject: [PATCH 19/86] Add institution --- docs/ifb_core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ifb_core.md b/docs/ifb_core.md index 0434a09..3c1f546 100644 --- a/docs/ifb_core.md +++ b/docs/ifb_core.md @@ -1,6 +1,6 @@ # nf-core/configs: IFB core Configuration -All nf-core pipelines have been successfully configured for use on the IFB core at the insert institution here. +All nf-core pipelines have been successfully configured for use on the cluster of the IFB (Institut Francais de Bioinformatique). To use, run the pipeline with `-profile ifb_core`. This will download and launch the [`ifb_core.config`](../conf/ifb_core.config) which has been pre-configured with a setup suitable for the IFB core cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. From e3216ccf79aa60edcb8edd30fa8e2618171de90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 16:00:56 +0100 Subject: [PATCH 20/86] typo --- docs/ifb_core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ifb_core.md b/docs/ifb_core.md index 3c1f546..90c0eb1 100644 --- a/docs/ifb_core.md +++ b/docs/ifb_core.md @@ -6,7 +6,7 @@ To use, run the pipeline with `-profile ifb_core`. This will download and launch ## How to use on IFB core -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on IFB core. You can do this by issuing the commands below: +Before running the pipeline you will need to load Nextflow using the environment module system on IFB core. You can do this by issuing the commands below: ```bash # Login to a compute node From d4cd07c646dabcbaa70ccdd3c02647f8c1b34e86 Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Wed, 17 Mar 2021 09:21:26 +0000 Subject: [PATCH 21/86] Typo in beforescript --- conf/eddie.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/eddie.config b/conf/eddie.config index 6145908..17ddbba 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -22,7 +22,7 @@ process { beforeScript = """ - '. /etc/profile.d/modules.sh + . /etc/profile.d/modules.sh sleep 2 module load 'anaconda/5.3.1' module load 'roslin/singularity/3.5.3' From 2ce1e2a9cc10ef5f1fc8389fd31e6f4fb7eab402 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 17 Mar 2021 12:32:47 +0000 Subject: [PATCH 22/86] Create rnaseq.config --- pipeline/rnaseq.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 pipeline/rnaseq.config diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pipeline/rnaseq.config @@ -0,0 +1 @@ + From 4ad6bb2bec13d3bf7d4d256c6e2f23e789e5c92d Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 17 Mar 2021 14:50:40 +0000 Subject: [PATCH 23/86] Singularity info --- docs/eddie.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index e6a2f87..fec6539 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -11,10 +11,10 @@ There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https: To use, run the pipeline with `-profile eddie` (one hyphen). This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). -The configuration file supports running nf-core pipelines with either a Conda environment or Docker containers running under Singularity. +The configuration file supports running nf-core pipelines with Docker containers running under Singularity. Support for Conda will follow. ```bash -nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie,singularity # ..rest of pipeline flags ``` Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. @@ -30,11 +30,19 @@ To load the most recent version: module load igmm/apps/nextflow ``` - This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. To set up Nextflow on a login node ... TODO +## Singularity set-up + +Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. + +``` +module load singularity +export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity/nf-core-rnaseq_v3.0" +``` + ## Using iGenomes references A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. From 059d353a1e1d14c4307c5427a65115c7d85b2d6e Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Thu, 18 Mar 2021 12:31:56 +0000 Subject: [PATCH 24/86] Removed Conda, added automount Singularity --- conf/eddie.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 17ddbba..70e0dcc 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -23,8 +23,6 @@ process { beforeScript = """ . /etc/profile.d/modules.sh - sleep 2 - module load 'anaconda/5.3.1' module load 'roslin/singularity/3.5.3' export SINGULARITY_TMPDIR="\$TMPDIR" """ @@ -46,4 +44,7 @@ env { singularity { envWhitelist = "SINGULARITY_TMPDIR" runOptions = '-p' -} \ No newline at end of file + enabled = true + autoMounts = true +} + From af6a6b1060d5b59c7e7be33b832701ada7ce2c27 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 18 Mar 2021 12:48:48 +0000 Subject: [PATCH 25/86] Details on running on eddie Removed Conda support, Singularity by default. Added login and screen session on wild west node options. --- docs/eddie.md | 60 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index fec6539..faecad7 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -11,13 +11,14 @@ There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https: To use, run the pipeline with `-profile eddie` (one hyphen). This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). -The configuration file supports running nf-core pipelines with Docker containers running under Singularity. Support for Conda will follow. +The configuration file supports running nf-core pipelines with Docker containers running under Singularity by default. Conda is not currently supported. ```bash -nextflow run nf-core/PIPELINE -profile eddie,singularity # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags ``` -Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. +If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. To list versions: @@ -30,19 +31,60 @@ To load the most recent version: module load igmm/apps/nextflow ``` -This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. +This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. -To set up Nextflow on a login node ... TODO +## Singularity set-up -## Singularity set-up +Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. -Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. - -``` +```bash module load singularity export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity/nf-core-rnaseq_v3.0" ``` +Singularity will create a directory `.singularity` in your `$HOME` directory on eddie. Space on `$HOME` is very limited, so it is a good idea to create a directory somewhere else with more room and link the locations. + +```bash +cd $HOME +mkdir /exports/eddie/path/to/my/area/.singularity +ln -s /exports/eddie/path/to/my/area/.singularity .singularity +``` + +## Running Nextflow + +### On a login node + +You can use a qlogin to run Nextflow, if you request more than the default 2GB of memory. Unfortunately you can't submit the initial Nextflow run process as a job as you can't qsub within a qsub. + +```bash +qlogin -l h_vmem=8G +``` + +If your eddie terminal disconnects your Nextflow job will stop. You can run Nextflow as a bash script on the command line using `nohup` to prevent this. + +``` +nohup ./nextflow_run.sh & +``` + +### On a wild west node + +Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). + +Start a new screen session. +```bash +screen -S +``` + +List existing screen sessions +```bash +screen -ls +``` + +Reconnect to an existing screen session +```bash +screen -r +``` + ## Using iGenomes references A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. From b48136a62e547aa2c8f337f76d4872ef4b80a224 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 18 Mar 2021 12:55:03 +0000 Subject: [PATCH 26/86] Added pipeline-specific config files --- docs/eddie.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index faecad7..5a8dfa5 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -14,7 +14,7 @@ This will download and launch the [`eddie.config`](../conf/eddie.config) which h The configuration file supports running nf-core pipelines with Docker containers running under Singularity by default. Conda is not currently supported. ```bash -nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie # ...rest of pipeline flags ``` Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. @@ -33,6 +33,14 @@ module load igmm/apps/nextflow This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. +### Pipeline-specific config files + +In addition to the generic nf-core configuration accessed via `-profile eddie`, there are pipeline-specific configuration files in the NextGenResources fileset to handle processes that are known to require specific custom configuration on the eddie system, in particular Java processes which require an extra memory overhead. For example: + +```bash +nextflow run nf-core/rnaseq -profile eddie -c /exports/igmm/eddie/NextGenResources/nextflow/conf/eddie.rnaseq.config # ...rest of pipeline flags +``` + ## Singularity set-up Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. From 93bde78faad71fbb97795b9fdbae1af79baa8f30 Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Sun, 21 Mar 2021 06:47:52 -0400 Subject: [PATCH 27/86] added JAX conf --- .github/workflows/main.yml | 2 +- README.md | 1 + nfcore_custom.config | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf00bea..41601d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 45721e8..5616780 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Currently documentation is available for the following systems: * [GOOGLE](docs/google.md) * [HEBBE](docs/hebbe.md) * [ICR_DAVROS](docs/icr_davros.md) +* [JAX](docs/jax.md) * [KRAKEN](docs/kraken.md) * [MPCDF](docs/mpcdf.md) * [MUNIN](docs/munin.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index a4c9939..3cafe07 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -46,6 +46,7 @@ profiles { uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } + jax { includeConfig "${params.custom_config_base}/conf/jax.config" } } // If user hostnames contain one of these substring and they are From 30c864e1121e5b5fcea5633eaa689498745670b2 Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Sun, 21 Mar 2021 06:49:11 -0400 Subject: [PATCH 28/86] added JAX conf --- conf/jax.config | 21 +++++++++++++++++++++ docs/jax.md | 8 ++++++++ 2 files changed, 29 insertions(+) create mode 100644 conf/jax.config create mode 100644 docs/jax.md diff --git a/conf/jax.config b/conf/jax.config new file mode 100644 index 0000000..a5437e9 --- /dev/null +++ b/conf/jax.config @@ -0,0 +1,21 @@ +params { + config_profile_description = 'The Jackson Laboratory Sumner HPC profile provided by nf-core/configs.' + config_profile_contact = 'Asaf Peer (@peera)' + config_profile_url = 'https://jacksonlaboratory.sharepoint.com/sites/ResearchIT/SitePages/Welcome-to-Sumner.aspx' + } + +executor.$slurm.queueSize = 250 +process { + executor = "slurm" + queue = "compute" + clusterOptions = {task.time < 72.h ? '-q batch' : '-q long'} + module = "slurm" + beforeScript = 'module load singularity' +} +singularity.enabled = true +singularity.autoMounts = true +params { + max_memory = 768.GB + max_cpus = 70 + max_time = 336.h + } diff --git a/docs/jax.md b/docs/jax.md new file mode 100644 index 0000000..d38cfbe --- /dev/null +++ b/docs/jax.md @@ -0,0 +1,8 @@ +# nf-core/configs: JAX Configuration + +All nf-core pipelines have been successfully configured for use on the JAX Sumner cluster at The Jackson Laboratory. + +To use, run the pipeline with `-profile jax`. This will download and launch the [`jax.config`](../conf/jax.config) which has been pre-configured with a setup suitable for JAX Sumner cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline and slurm will be used as well. + +>NB: You will need an account to use the HPC cluster JAX in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow should not be executed on the login nodes. If in doubt contact IT. From 4570718b12ef8b5b1e781a94bd3a270167d3d004 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Mar 2021 21:45:39 -0500 Subject: [PATCH 29/86] fix(ganymede): Add TMPDIR and SINGULARITY_CACHEDIR env variables --- conf/utd_ganymede.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 409dc0a..f701a3c 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -5,6 +5,11 @@ params { config_profile_url = 'http://docs.oithpc.utdallas.edu/' } +env { + TMPDIR = '/home/$USER/scratch/tmp' + SINGULARITY_CACHEDIR = '/home/$USER/scratch/tmp' +} + singularity { enabled = true envWhitelist='SINGULARITY_BINDPATH' From 34ccce9531351fd8133c3dc3a77dab20398a5735 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Mar 2021 21:46:25 -0500 Subject: [PATCH 30/86] fix(ganymede): Add process resources Trimgalore though it can be a bottom neck, doesn't require the 1 big memory node we have and can be run on the genomics queue. --- conf/utd_ganymede.config | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index f701a3c..c64e673 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -20,6 +20,28 @@ process { beforeScript = 'module load singularity/3.2.1' executor = 'slurm' queue = { task.memory >= 32.GB && task.cpu <= 12 ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' } + + withName:TRIMGALORE { + memory = 31.GB + } + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 16 * task.attempt, 'cpus' ) } + memory = { check_max( 31.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_large { + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 16.h * task.attempt, 'time' ) } + } } params { From a6975cdd4ef05a552aaa6de492d88a525974a313 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 22 Mar 2021 12:42:56 -0500 Subject: [PATCH 31/86] fix(ganymede): large => high Co-authored-by: drpatelh --- conf/utd_ganymede.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index c64e673..01bb10b 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -37,9 +37,9 @@ process { time = { check_max( 8.h * task.attempt, 'time' ) } } - withLabel:process_large { + withLabel:process_high { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 1.GB * task.attempt, 'memory' ) } + memory = { check_max( 120.GB * task.attempt, 'memory' ) } time = { check_max( 16.h * task.attempt, 'time' ) } } } From 791f3c95f34eb21d2ecd573b8611ba980f6bf0a3 Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 23 Mar 2021 16:07:07 +0000 Subject: [PATCH 32/86] Pipeline specific configs moving to confs/pipeline --- docs/eddie.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 5a8dfa5..1a63568 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -33,14 +33,6 @@ module load igmm/apps/nextflow This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. -### Pipeline-specific config files - -In addition to the generic nf-core configuration accessed via `-profile eddie`, there are pipeline-specific configuration files in the NextGenResources fileset to handle processes that are known to require specific custom configuration on the eddie system, in particular Java processes which require an extra memory overhead. For example: - -```bash -nextflow run nf-core/rnaseq -profile eddie -c /exports/igmm/eddie/NextGenResources/nextflow/conf/eddie.rnaseq.config # ...rest of pipeline flags -``` - ## Singularity set-up Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. From da8cf2f20705f4fca348c8a6185dada9430092b4 Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Wed, 24 Mar 2021 10:46:26 +0000 Subject: [PATCH 33/86] Added pipeline-specific config files for rnaseq and sarek --- conf/pipeline/rnaseq/eddie.config | 15 +++++++++++++++ conf/pipeline/sarek/eddie.config | 27 +++++++++++++++++++++++++++ pipeline/rnaseq.config | 13 +++++++++++++ pipeline/sarek.config | 1 + 4 files changed, 56 insertions(+) create mode 100644 conf/pipeline/rnaseq/eddie.config create mode 100644 conf/pipeline/sarek/eddie.config create mode 100644 pipeline/rnaseq.config diff --git a/conf/pipeline/rnaseq/eddie.config b/conf/pipeline/rnaseq/eddie.config new file mode 100644 index 0000000..d8e76d0 --- /dev/null +++ b/conf/pipeline/rnaseq/eddie.config @@ -0,0 +1,15 @@ +process { + +withName : "PICARD_MARKDUPLICATES" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +withName : "QUALIMAP_RNASEQ" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +withName : "FASTQC" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +} diff --git a/conf/pipeline/sarek/eddie.config b/conf/pipeline/sarek/eddie.config new file mode 100644 index 0000000..3e1b811 --- /dev/null +++ b/conf/pipeline/sarek/eddie.config @@ -0,0 +1,27 @@ +process { + + withName:MapReads { + cpus = 16 + } + withName:BuildDict { + cpus = 1 + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:BamQC { + cpus = 8 + memory = 128.GB + clusterOptions = {"-l h_vmem=${(task.memory + 8.GB).bytes/task.cpus}"} + } + withName:MarkDuplicates { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:BaseRecalibrator { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:ApplyBQSR { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:GatherBQSRReports { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } +} diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config new file mode 100644 index 0000000..1a27463 --- /dev/null +++ b/pipeline/rnaseq.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/rnaseq custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/rnaseq folder and imported + * under a profile name here. + */ + +profiles { + eddie { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/eddie.config" } +} diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 4ef48da..d781478 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -12,4 +12,5 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" } + eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" } } \ No newline at end of file From 96fe6185a08cb12de4c510c22ba84974ba286f72 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:01:41 +0000 Subject: [PATCH 34/86] IGMM-specific docs --- docs/eddie.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 1a63568..363dcc5 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -35,7 +35,9 @@ This config enables Nextflow to manage the pipeline jobs via the SGE job schedul ## Singularity set-up -Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. +Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. + +If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area which has limited space. It will take time to download all the Singularity containers, but you can use this again. ```bash module load singularity @@ -66,7 +68,7 @@ If your eddie terminal disconnects your Nextflow job will stop. You can run Next nohup ./nextflow_run.sh & ``` -### On a wild west node +### On a wild west node - IGMM only Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). @@ -87,7 +89,7 @@ screen -r ## Using iGenomes references -A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. +A local copy of the iGenomes resource has been made available on the Eddie HPC for those with access to `/exports/igmm/eddie/NextGenResources` so you should be able to run the pipeline against any reference available in the `igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Adjusting maximum resources From 2a8ae2b2148c3782b601f1b613f3b1462fb8384f Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:12:41 +0000 Subject: [PATCH 35/86] Update eddie.md --- docs/eddie.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index 363dcc5..c51f7c1 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -37,7 +37,7 @@ This config enables Nextflow to manage the pipeline jobs via the SGE job schedul Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. -If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area which has limited space. It will take time to download all the Singularity containers, but you can use this again. +If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area (which has limited space). It will take time to download all the Singularity containers, but you can use this again. ```bash module load singularity From aeb0ffcfdea232d81edc5c7a635f579235323247 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:26:09 +0000 Subject: [PATCH 36/86] Spacing fix Co-authored-by: Maxime Garcia --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index b338f1c..4fffc74 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 158402eddf6784d0e7d292d3bc8a76d26941d2fe Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 24 Mar 2021 12:26:49 +0100 Subject: [PATCH 37/86] Update nfcore_custom.config --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 4fffc74..9ea0197 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 8e6223d1bdb341b5b00a990027a29c371ff5ed99 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 24 Mar 2021 12:36:50 +0100 Subject: [PATCH 38/86] Update nfcore_custom.config --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 907e561..eb287b9 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From 2d90c919aad00e682a7bb78c50ee6f265e00064a Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:51:37 +0000 Subject: [PATCH 39/86] Added eddie profile to github workflow tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41601d4..50dc704 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From a0d42f51536cbc21323c95afce0f19e9e7c1aa4a Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:55:31 +0000 Subject: [PATCH 40/86] Fixed code blocks for markdown linting --- docs/eddie.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index c51f7c1..283e21f 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -27,6 +27,7 @@ module avail igmm/apps/nextflow ``` To load the most recent version: + ```bash module load igmm/apps/nextflow ``` @@ -64,7 +65,7 @@ qlogin -l h_vmem=8G If your eddie terminal disconnects your Nextflow job will stop. You can run Nextflow as a bash script on the command line using `nohup` to prevent this. -``` +```bash nohup ./nextflow_run.sh & ``` @@ -73,16 +74,19 @@ nohup ./nextflow_run.sh & Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). Start a new screen session. + ```bash screen -S ``` List existing screen sessions + ```bash screen -ls ``` Reconnect to an existing screen session + ```bash screen -r ``` From 3a8e54f153f249f7ed06dd9e6ea1e47eec137ca1 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:58:10 +0000 Subject: [PATCH 41/86] Removed sneaky trailing whitespace. --- docs/eddie.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 283e21f..cb78d71 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -17,8 +17,7 @@ The configuration file supports running nf-core pipelines with Docker containers nextflow run nf-core/PIPELINE -profile eddie # ...rest of pipeline flags ``` -Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. -If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. To list versions: From 922ab87eb0f69cf994220a4f15f008f9784d16fa Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Wed, 24 Mar 2021 11:25:47 -0400 Subject: [PATCH 42/86] added jax singularity cache dir --- conf/jax.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/jax.config b/conf/jax.config index a5437e9..7cf790e 100644 --- a/conf/jax.config +++ b/conf/jax.config @@ -2,6 +2,7 @@ params { config_profile_description = 'The Jackson Laboratory Sumner HPC profile provided by nf-core/configs.' config_profile_contact = 'Asaf Peer (@peera)' config_profile_url = 'https://jacksonlaboratory.sharepoint.com/sites/ResearchIT/SitePages/Welcome-to-Sumner.aspx' + singularity_cache_dir = '/fastscratch/singularity_cache_nfcore' } executor.$slurm.queueSize = 250 @@ -12,8 +13,11 @@ process { module = "slurm" beforeScript = 'module load singularity' } -singularity.enabled = true -singularity.autoMounts = true +singularity{ + enabled = true + autoMounts = true + cacheDir = params.singularity_cache_dir +} params { max_memory = 768.GB max_cpus = 70 From e2c583d5581ce40d18505d3af4155a85869fe40c Mon Sep 17 00:00:00 2001 From: Santiago Lacalle <33420220+slacalle@users.noreply.github.com> Date: Thu, 25 Mar 2021 12:38:15 +0000 Subject: [PATCH 43/86] Remove "/rdsgpfs" "/rdsgpfs" symlink will not be present on all nodes. Without that mount the container creation will fail. --- conf/imperial.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/imperial.config b/conf/imperial.config index f40d92b..eeb98ae 100644 --- a/conf/imperial.config +++ b/conf/imperial.config @@ -27,7 +27,7 @@ executor { singularity { enabled = true autoMounts = true - runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp" + runOptions = "-B /rds/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp" } process { From eccc9ea8d5d91e29ac0b981b0dfe6c7c604bb555 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 26 Mar 2021 15:20:32 +0100 Subject: [PATCH 44/86] Define params.project and params.clusterOptions in uppmax config --- conf/uppmax.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index bc3d8c4..ff03ef1 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -1,8 +1,10 @@ -//Profile config names for nf-core/configs +// Profile config names for nf-core/configs params { config_profile_description = 'Swedish UPPMAX cluster profile provided by nf-core/configs.' config_profile_contact = 'Phil Ewels (@ewels)' config_profile_url = 'https://www.uppmax.uu.se/' + project = null + clusterOptions = null } singularity { From d34bc40841880062923d3a2e0595648640a1da49 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 26 Mar 2021 15:22:57 +0100 Subject: [PATCH 45/86] UPPMAX: add to schema_ignore_params --- conf/uppmax.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/uppmax.config b/conf/uppmax.config index ff03ef1..3b518d4 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -5,6 +5,7 @@ params { config_profile_url = 'https://www.uppmax.uu.se/' project = null clusterOptions = null + schema_ignore_params = "genomes,input_paths,cluster-options,clusterOptions,project" } singularity { From 8fc60e2e70bee84a28a72d0c82eb1fead3f1aff3 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 29 Mar 2021 19:25:58 +0200 Subject: [PATCH 46/86] add config for biohpc_gen --- .github/workflows/main.yml | 2 +- README.md | 1 + conf/biohpc_gen.config | 24 ++++++++++++++++++++++++ docs/biohpc_gen.md | 16 ++++++++++++++++ nfcore_custom.config | 3 ++- 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100755 conf/biohpc_gen.config create mode 100644 docs/biohpc_gen.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 50dc704..38adb48 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 5616780..4217d62 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ Currently documentation is available for the following systems: * [BIGPURPLE](docs/bigpurple.md) * [BI](docs/bi.md) * [BINAC](docs/binac.md) +* [BIOHPC_GEN](docs/biohpc_gen.md) * [CBE](docs/cbe.md) * [CCGA_DX](docs/ccga_dx.md) * [CCGA_MED](docs/ccga_med.md) diff --git a/conf/biohpc_gen.config b/conf/biohpc_gen.config new file mode 100755 index 0000000..cdd2c03 --- /dev/null +++ b/conf/biohpc_gen.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'BioHPC Genomics (biohpc_gen) cluster profile provided by nf-core/configs' + config_profile_contact = 'Patrick Hüther (@phue)' + config_profile_url = 'https://collab.lmu.de/display/BioHPCGenomics/BioHPC+Genomics' +} + +process { + executor = 'slurm' + queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } + beforeScript = 'module use /dss/dssfs01/pr53da/pr53da-dss-0000/spack/modules/x86_avx2/linux*' + module = 'charliecloud/0.22:miniconda3' + clusterOptions = '-M biohpc_gen' +} + +charliecloud { + enabled = true +} + +params { + params.max_time = 14.d + params.max_cpus = 80 + params.max_memory = 3.TB +} \ No newline at end of file diff --git a/docs/biohpc_gen.md b/docs/biohpc_gen.md new file mode 100644 index 0000000..66647ac --- /dev/null +++ b/docs/biohpc_gen.md @@ -0,0 +1,16 @@ +# nf-core/configs: BioHPC Genomics (BIOHPC_GEN) Configuration + +All nf-core pipelines have been successfully configured for use on the BioHPC Genomics (biohpc_gen) cluster that is housed at the Leibniz Rechenzentrum (LRZ) for research groups at the Faculty of Biology of the Ludwig-Maximilians-University (LMU) in Munich. + +To use, run the pipeline with `-profile biohpc_gen`. This will download and launch the [`biohpc_gen.config`](../conf/biohpc_gen.config) which has been pre-configured with a setup suitable for the biohpc_gen cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Charliecloud container before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Charliecloud using the environment module system on biohpc_gen. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Charliecloud environment modules +module purge +module load nextflow charliecloud/0.22 +``` + +>NB: You will need an account to use the LRZ Linux cluster as well as group access to the biohpc_gen cluster in order to run nf-core pipelines. +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. diff --git a/nfcore_custom.config b/nfcore_custom.config index eb287b9..807b6f7 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -15,6 +15,7 @@ profiles { bi { includeConfig "${params.custom_config_base}/conf/bi.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } + biohpc_gen { includeConfig "${params.custom_config_base}/conf/biohpc_gen.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } ccga_med { includeConfig "${params.custom_config_base}/conf/ccga_med.config" } @@ -23,7 +24,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From 7c5a0871a3582f8fefe3ab6c76ec4c8a2479b5fe Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 29 Mar 2021 19:36:49 +0200 Subject: [PATCH 47/86] add note about required nextflow version --- docs/biohpc_gen.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/biohpc_gen.md b/docs/biohpc_gen.md index 66647ac..1078835 100644 --- a/docs/biohpc_gen.md +++ b/docs/biohpc_gen.md @@ -12,5 +12,6 @@ module purge module load nextflow charliecloud/0.22 ``` +>NB: Charliecloud support requires Nextflow version `21.03.0-edge` or later. >NB: You will need an account to use the LRZ Linux cluster as well as group access to the biohpc_gen cluster in order to run nf-core pipelines. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. From 700776f759b59fb628dd046cee3ff389cac5987f Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 30 Mar 2021 16:11:15 +0100 Subject: [PATCH 48/86] Change to shared Singularity cache --- docs/eddie.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index cb78d71..e5dbe5c 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -35,13 +35,13 @@ This config enables Nextflow to manage the pipeline jobs via the SGE job schedul ## Singularity set-up -Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. +Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path below. If some containers for your pipeline run are not present, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have them added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area (which has limited space). It will take time to download all the Singularity containers, but you can use this again. ```bash module load singularity -export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity/nf-core-rnaseq_v3.0" +export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity" ``` Singularity will create a directory `.singularity` in your `$HOME` directory on eddie. Space on `$HOME` is very limited, so it is a good idea to create a directory somewhere else with more room and link the locations. From adfb5366b93c5a03af86c147d369d222add6cc8d Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 30 Mar 2021 16:15:17 +0100 Subject: [PATCH 49/86] Java overhead process list updated Added HaplotypeCaller and GenotypeGVCFs to set of processes that need Java overhead --- conf/pipeline/sarek/eddie.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/pipeline/sarek/eddie.config b/conf/pipeline/sarek/eddie.config index 3e1b811..33c4e02 100644 --- a/conf/pipeline/sarek/eddie.config +++ b/conf/pipeline/sarek/eddie.config @@ -24,4 +24,10 @@ process { withName:GatherBQSRReports { clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} } + withName:HaplotypeCaller { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:GenotypeGVCFs { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } } From e5f2a63079f644894a11cb42b3aa16dfd36d40ba Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 30 Mar 2021 16:21:53 +0100 Subject: [PATCH 50/86] Java overhead processes list updated Added all processes calling GATK for Mutect2 variant calling --- conf/pipeline/sarek/eddie.config | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/conf/pipeline/sarek/eddie.config b/conf/pipeline/sarek/eddie.config index 33c4e02..9629337 100644 --- a/conf/pipeline/sarek/eddie.config +++ b/conf/pipeline/sarek/eddie.config @@ -30,4 +30,22 @@ process { withName:GenotypeGVCFs { clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} } + withName:Mutect2 { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:MergeMutect2Stats { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:PileupSummariesForMutect2 { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:MergePileupSummaries { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:CalculateContamination { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:FilterMutect2Calls { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } } From 6e201a71f44ea928ab367f9ae989b620517aab25 Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 30 Mar 2021 16:31:00 +0100 Subject: [PATCH 51/86] ATAC-seq pipeline config Java processes with overhead memory requirements --- conf/pipeline/atacseq/eddie.config | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 conf/pipeline/atacseq/eddie.config diff --git a/conf/pipeline/atacseq/eddie.config b/conf/pipeline/atacseq/eddie.config new file mode 100644 index 0000000..5e2b96b --- /dev/null +++ b/conf/pipeline/atacseq/eddie.config @@ -0,0 +1,14 @@ +process { + withName : "FASTQC" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName : "MERGED_LIB_BAM" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName : "MERGED_LIB_PICARD_METRICS" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName : "MERGED_REP_BAM" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } +} From 60ae267e4397cc9664154ad5cd5df19692a9b615 Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 6 Apr 2021 11:35:01 +0200 Subject: [PATCH 52/86] use SLURM_CLUSTERS env var this seems to be more reliable for queue checking than using the clusterOptions directive. See https://github.com/nextflow-io/nextflow/issues/807 --- conf/biohpc_gen.config | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conf/biohpc_gen.config b/conf/biohpc_gen.config index cdd2c03..b625b20 100755 --- a/conf/biohpc_gen.config +++ b/conf/biohpc_gen.config @@ -5,12 +5,15 @@ params { config_profile_url = 'https://collab.lmu.de/display/BioHPCGenomics/BioHPC+Genomics' } +env { + SLURM_CLUSTERS='biohpc_gen' +} + process { executor = 'slurm' queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } beforeScript = 'module use /dss/dssfs01/pr53da/pr53da-dss-0000/spack/modules/x86_avx2/linux*' module = 'charliecloud/0.22:miniconda3' - clusterOptions = '-M biohpc_gen' } charliecloud { @@ -21,4 +24,4 @@ params { params.max_time = 14.d params.max_cpus = 80 params.max_memory = 3.TB -} \ No newline at end of file +} From d69af0f287700e82680a958295d41c3aa231b086 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Tue, 6 Apr 2021 16:42:22 +0200 Subject: [PATCH 53/86] Merging --- conf/pipeline/eager/eva.config | 14 +++++++------- conf/pipeline/eager/shh.config | 6 ++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 8850cf1..46fddae 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -63,19 +63,19 @@ process { } withName: fastqc { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } } withName: adapter_removal { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(12000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(12000).toString().replaceAll(/[\sB]/,'')}M" } } withName: dedup { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } } withName: markduplicates { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(8000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(8000).toString().replaceAll(/[\sB]/,'')}M" } } withName: malt { @@ -99,11 +99,11 @@ process { } withName: qualimap { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M" } } withName: damageprofiler { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M" } } withName: circularmapper { @@ -115,7 +115,7 @@ process { } withName: preseq { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } } } diff --git a/conf/pipeline/eager/shh.config b/conf/pipeline/eager/shh.config index 44d1c3c..5046df0 100644 --- a/conf/pipeline/eager/shh.config +++ b/conf/pipeline/eager/shh.config @@ -20,6 +20,12 @@ process { queue = { task.memory > 756.GB ? 'supercruncher' : 'long' } } + withName: circulargenerator { + cpus = { check_max( 1, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } + withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } From be1d35840d65ac0dfeccc13c511d99d2236f0f20 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Tue, 6 Apr 2021 17:48:16 +0200 Subject: [PATCH 54/86] Add EVA and EAGER@EVA --- .github/workflows/main.yml | 2 +- conf/eva.config | 2 +- conf/pipeline/eager/eva.config | 37 ++++++++++++++++++---------------- docs/eva.md | 2 +- docs/pipeline/eager/eva.md | 2 +- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c500dfb..96776a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/conf/eva.config b/conf/eva.config index 497085d..c0a1d4f 100644 --- a/conf/eva.config +++ b/conf/eva.config @@ -49,4 +49,4 @@ profiles { debug { cleanup = false } -} +} \ No newline at end of file diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 46fddae..2e07d57 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -9,10 +9,12 @@ params { // Specific nf-core/eager process configuration process { + beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"' + maxRetries = 2 // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion - clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toMega().toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" } withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } @@ -59,63 +61,64 @@ process { // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use withName: makeSeqDict { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -v JAVA_OPTS='-XX:ParallelGCThreads=1' -l h_vmem=${(task.memory.toGiga() + 3)}G,virtual_free=${(task.memory.toGiga() + 3)}G" } } withName: fastqc { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: adapter_removal { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(12000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(12000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: dedup { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: markduplicates { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(8000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(8000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } } withName: malt { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: maltextract { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: multivcfanalyzer { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: mtnucratio { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: vcf2genome { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: qualimap { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } } withName: damageprofiler { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(5000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } } withName: circularmapper { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: circulargenerator { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(1000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } } withName: preseq { - clusterOptions = { "-S /bin/bash -l h_vmem=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M,virtual_free=${task.memory.toMega().plus(4000).toString().replaceAll(/[\sB]/,'')}M" } + clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' } } } @@ -209,4 +212,4 @@ profiles { bwaalnl = 16500 } } -} +} \ No newline at end of file diff --git a/docs/eva.md b/docs/eva.md index 42575b2..9b83222 100644 --- a/docs/eva.md +++ b/docs/eva.md @@ -25,4 +25,4 @@ Note the following characteristics of this profile: ### debug -This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen` +This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen` \ No newline at end of file diff --git a/docs/pipeline/eager/eva.md b/docs/pipeline/eager/eva.md index a8dc563..71f4a2e 100644 --- a/docs/pipeline/eager/eva.md +++ b/docs/pipeline/eager/eva.md @@ -31,4 +31,4 @@ Specific configurations for eva has been made for eager. #### Pathogen - `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) -- `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) +- `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) \ No newline at end of file From 504c10d6ff5d2965a55f282c81ae66325ae0d380 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 6 Apr 2021 17:50:25 +0200 Subject: [PATCH 55/86] Update README.md --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 56c73b6..687a1ae 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,20 @@ A repository for hosting Nextflow configuration files containing custom paramete ## Table of contents -- [Using an existing config](#using-an-existing-config) - - [Configuration and parameters](#configuration-and-parameters) - - [Offline usage](#offline-usage) -- [Adding a new config](#adding-a-new-config) - - [Checking user hostnames](#checking-user-hostnames) - - [Testing](#testing) - - [Documentation](#documentation) - - [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -- [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) - - [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) - - [Pipeline-specific documentation](#pipeline-specific-documentation) - - [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) - - [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) -- [Help](#help) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline-specific institutional documentation](#pipeline-specific-institutional-documentation) + * [Pipeline-specific documentation](#pipeline-specific-documentation) + * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) + * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) +* [Help](#help) ## Using an existing config From 60c66d93aca43d7ae3473ff182c9a83e087695cc Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Tue, 6 Apr 2021 17:53:27 +0200 Subject: [PATCH 56/86] Markdown linting --- docs/eva.md | 2 +- docs/pipeline/eager/eva.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/eva.md b/docs/eva.md index 9b83222..24a964c 100644 --- a/docs/eva.md +++ b/docs/eva.md @@ -25,4 +25,4 @@ Note the following characteristics of this profile: ### debug -This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen` \ No newline at end of file +This simple profile just turns off automatic clean up of intermediate files. This can be useful for debugging. Specify e.g. with `-profile eva,archgen`. diff --git a/docs/pipeline/eager/eva.md b/docs/pipeline/eager/eva.md index 71f4a2e..a8dc563 100644 --- a/docs/pipeline/eager/eva.md +++ b/docs/pipeline/eager/eva.md @@ -31,4 +31,4 @@ Specific configurations for eva has been made for eager. #### Pathogen - `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`) -- `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) \ No newline at end of file +- `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`) From 0ab9edde72bced10a8733e7936554ddf161b3b96 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 7 Apr 2021 09:24:30 +0100 Subject: [PATCH 57/86] Delete conf/pipeline/atacseq directory Pipeline-specific config not supported yet by ATAC-seq pipeline --- conf/pipeline/atacseq/eddie.config | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 conf/pipeline/atacseq/eddie.config diff --git a/conf/pipeline/atacseq/eddie.config b/conf/pipeline/atacseq/eddie.config deleted file mode 100644 index 5e2b96b..0000000 --- a/conf/pipeline/atacseq/eddie.config +++ /dev/null @@ -1,14 +0,0 @@ -process { - withName : "FASTQC" { - clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} - } - withName : "MERGED_LIB_BAM" { - clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} - } - withName : "MERGED_LIB_PICARD_METRICS" { - clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} - } - withName : "MERGED_REP_BAM" { - clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} - } -} From 53b54ba8454f02e55d799214c9bcf87fd8b0d2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrick=20H=C3=BCther?= Date: Wed, 7 Apr 2021 13:18:06 +0200 Subject: [PATCH 58/86] Remove whitespace Co-authored-by: Alexander Peltzer --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 807b6f7..6b51561 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,7 +24,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From ce465cca57113aa4cf568f86a35211b0f2d1e4d0 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 7 Apr 2021 19:27:24 +0200 Subject: [PATCH 59/86] Apply suggestions from code review Co-authored-by: Alexander Peltzer --- conf/eva.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/eva.config b/conf/eva.config index c0a1d4f..ad71e12 100644 --- a/conf/eva.config +++ b/conf/eva.config @@ -32,7 +32,6 @@ profiles { max_cpus = 32 max_time = 720.h //Illumina iGenomes reference file path - igenomes_base = "/projects1/public_data/igenomes/" } process { @@ -49,4 +48,4 @@ profiles { debug { cleanup = false } -} \ No newline at end of file +} From 7ac66dcd72cb4693a37f4e889ee5ca067028624b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 13 Apr 2021 17:07:02 +0100 Subject: [PATCH 60/86] Update crick.config --- conf/crick.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/crick.config b/conf/crick.config index 338bb76..f9c3197 100755 --- a/conf/crick.config +++ b/conf/crick.config @@ -17,7 +17,7 @@ process { params { max_memory = 224.GB max_cpus = 32 - max_time = 72.h + max_time = '72.h' igenomes_base = '/camp/svc/reference/Genomics/aws-igenomes' } From 0fc90586e04894e93a877416649ec4248173fadc Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Apr 2021 22:26:37 +0200 Subject: [PATCH 61/86] Update mpcdf.md --- docs/mpcdf.md | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index af16d76..0d48dd2 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -4,18 +4,17 @@ All nf-core pipelines have been successfully configured for use on the HPCs at [ > :warning: these profiles are not officially supported by the MPCDF. -To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile ,mpcdf`. +To run Nextflow, the `jdk` module must be loaded. To use the nf-core profile(s), run the pipeline with `-profile mpcdf,`. + +Currently profiles for the following clusters are supported: `cobra`, `raven` + +All profiles use `singularity` as the corresponding containerEngine. To prevent repeatedly downloading the same singularity image for every pipeline run, for all profiles we recommend specifying a cache location in your `~/.bash_profile` with the `$NXF_SINGULARITY_CACHEDIR` bash variable. -Currently the following clusters are supported: cobra, raven >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). ## cobra -Cobra does not currently support singularity, therefore the anaconda/module is loaded for each process. - -Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). - To use: `-profile cobra,mpcdf` Sets the following parameters: @@ -31,10 +30,6 @@ Sets the following parameters: ## raven -Raven does not currently support singularity, therefore `module load anaconda/3/2020.02` is loaded for each process. - -Due to this, we also recommend setting the `$NXF_CONDA_CACHEDIR` to a location of your choice to store all environments (so to prevent nextflow building the environment on every run). - To use: `-profile raven,mpcdf` Sets the following parameters: From b9a00f79268c237a540060e7a43c2c8418c68bb8 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Apr 2021 22:27:07 +0200 Subject: [PATCH 62/86] Update mpcdf.config --- conf/mpcdf.config | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/conf/mpcdf.config b/conf/mpcdf.config index 481ebc4..c6c3c9c 100644 --- a/conf/mpcdf.config +++ b/conf/mpcdf.config @@ -1,24 +1,28 @@ params { config_profile_description = 'MPCDF HPC profiles (unoffically) provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_url = 'https://www.mpcdf.mpg.de/services/computing' + config_profile_url = 'https://www.mpcdf.mpg.de/services/supercomputing' } profiles { cobra { - // Does not have singularity! Conda module must be used, but it is - // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} - // To create common cache dir process { - beforeScript = 'module load anaconda/3/2020.02' + beforeScript = 'module load singularity' executor = 'slurm' } executor { queueSize = 8 } + + // Set $NXF_SINGULARITY_CACHEDIR in your ~/.bash_profile + // to stop downloading the same image for every run + singularity { + enabled = true + autoMounts = true + } params { config_profile_description = 'MPCDF cobra profile (unofficially) provided by nf-core/configs.' @@ -28,18 +32,22 @@ profiles { } } raven { - // Does not have singularity! Conda module must be used, but it is - // recommended to set NXF_CONDA_CACHEDIR var in ~/.bash{_profile,rc} - // to create common cache dir process { - beforeScript = 'module load anaconda/3/2020.02' + beforeScript = 'module load singularity' executor = 'slurm' } executor { queueSize = 8 } + + // Set $NXF_SINGULARITY_CACHEDIR in your ~/.bash_profile + // to stop downloading the same image for every run + singularity { + enabled = true + autoMounts = true + } params { config_profile_description = 'MPCDF raven profile (unofficially) provided by nf-core/configs.' @@ -47,5 +55,8 @@ profiles { max_cpus = 192 max_time = 24.h } + } + debug { + cleanup = false } } From 12a451880306262049b068722af4b52956eddeac Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Apr 2021 22:29:15 +0200 Subject: [PATCH 63/86] Markdown lint fix --- docs/mpcdf.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/mpcdf.md b/docs/mpcdf.md index 0d48dd2..73ed52f 100644 --- a/docs/mpcdf.md +++ b/docs/mpcdf.md @@ -10,7 +10,6 @@ Currently profiles for the following clusters are supported: `cobra`, `raven` All profiles use `singularity` as the corresponding containerEngine. To prevent repeatedly downloading the same singularity image for every pipeline run, for all profiles we recommend specifying a cache location in your `~/.bash_profile` with the `$NXF_SINGULARITY_CACHEDIR` bash variable. - >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. Check the [MPCDF documentation](https://www.mpcdf.mpg.de/services/computing). ## cobra From 10abca712bb9f8c6ba009fbd51b2efb709b68d49 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 16 Apr 2021 17:01:58 +0200 Subject: [PATCH 64/86] fix: first draft of bianca guide --- docs/uppmax.md | 174 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 167 insertions(+), 7 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index d9ac5ce..c8375a2 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -19,7 +19,7 @@ You can do this with the `--project` flag (two hyphens) when launching nextflow. nextflow run nf-core/PIPELINE -profile uppmax --project snic2018-1-234 # ..rest of pipeline flags ``` -> NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. +$ NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. Before running the pipeline you will need to either install Nextflow or load it using the environment module system. @@ -43,7 +43,7 @@ If your nf-core pipeline run is running out of memory, you can run on a fat node ``` This raises the ceiling of available memory from the default of `128.GB` to `256.GB`. -Rackham has nodes with 128GB, 256GB and 1TB memory available. +`Rackham` has nodes with 128GB, 256GB and 1TB memory available. Note that each job will still start with the same request as normal, but restarted attempts with larger requests will be able to request greater amounts of memory. @@ -76,12 +76,172 @@ To use it, submit with `-profile uppmax,devel`. ## Running on Bianca -For security reasons, there is no internet access on Bianca so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on Bianca you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your Bianca project. +$ :warning: For more information about `bianca`, follow the [UPPMAX `bianca` user guide](http://uppmax.uu.se/support/user-guides/bianca-user-guide/). +$ :warning: For more information, follow the [nf-core guide for running offline](https://nf-co.re/usage/offline) and the [nf-core `tools` guide for downloading pipelines for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). + > :warning: For more information about using `Singularity` with UPPMAX, follow the [UPPMAX `Singularity` guide](https://www.uppmax.uu.se/support-sv/user-guides/singularity-user-guide/). -You can follow the guide for downloading pipelines [for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). Note that you will have to download the singularity images as well. +For security reasons, there is no internet access on `bianca` so you can't download from or upload files to the cluster directly. +Before running a nf-core pipeline on `bianca` you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your own `bianca` project. -Next transfer the pipeline and the singularity images to your project. Before running the pipeline you will have to indicate to nextflow where the singularity images are located by setting `NXF_SINGULARITY_CACHEDIR` : +In this guide, we use `rackham` to download and transfer files to wharf, but it can also be done on your own computer. +If you use `rackham` to download the pipeline and the singularity containers, we recommend using an interactive session (cf [interactive guide](https://www.uppmax.uu.se/support/faq/running-jobs-faq/how-can-i-run-interactively-on-a-compute-node/)), which is what we do in the following guide. -`export NXF_SINGULARITY_CACHEDIR=Your_Location_For_The_Singularity_directory/.` +### Download Nextflow -You should now be able to run your nf-core pipeline on bianca. +You can use the UPPMAX provided `module`, but if necessary, you can also download a more recent version. + +```bash +# Connect to rackham +$ ssh -X @rackham.uppmax.uu.se +# Or stay in your terminal + +# Download the nextflow-all bundle +$ wget https://github.com/nextflow-io/nextflow/releases/download/v/nextflow--all + +# Connect to wharf using sftp +# For FileZilla follow the bianca user guide +$ sftp -@bianca-sftp.uppmax.uu.se:- + +# Transfer nextflow to wharf +sftp> put nextflow--all . + +# Exit sftp +$ exit + +# Connect to bianca +$ ssh -A -@bianca.uppmax.uu.se + +# Go to your project +$ cd /castor/project/proj_nobackup + +# Make folder for Nextflow +$ mkdir tools +$ mkdir tools/nextflow + +# Move Nextflow from wharf to its directory +$ mv /castor/project/proj_nobackup/wharf//-/nextflow--all /castor/project/proj_nobackup/tools/nextflow + +# Establish permission +$ chmod a+x /castor/project/proj_nobackup/tools/nextflow/nextflow--all + +# If you want other people to use it +# Be sure that your group has rights to the directory as well + +$ chown -R . /castor/project/proj_nobackup/tools/nextflow/nextflow--all + +# Make a link to it +$ ln -s /castor/project/proj_nobackup/tools/nextflow/nextflow--all /castor/project/proj_nobackup/tools/nextflow/nextflow + +# And every time you're launching Nextflow, don't forget to export the following ENV variables +# Or add them to your .bashrc file +$ export NXF_HOME=/castor/project/proj/nobackup/tools/nextflow/ +$ export PATH=${NXF_HOME}:${PATH} +$ export NXF_TEMP=$SNIC_TMP +$ export NXF_LAUNCHER=$SNIC_TMP +$ export NXF_SINGULARITY_CACHEDIR=/castor/project/proj_nobackup/singularity-images +``` + +### Download nf-core pipelines + +```bash +# Connect to rackham +$ ssh -X @rackham.uppmax.uu.se +# Or stay in your terminal + +# Open an interactive session (if you are on rackham) +$ interactive + +# Install the latest pip version +$ pip3 install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev --user + +# Download a pipeline (nf-core/rnaseq 3.0) with the singularity images +$ nf-core download rnaseq -r 3.0 -s -p 10 --compress none + +# Download a pipeline (nf-core/sarek 2.7) with the singularity images +$ nf-core download sarek -r 2.7 -s -p 10 --compress none + +# Download specific Singularity images +$ singularity pull --name nfcore-sareksnpeff-2.7.GRCh38.img docker://nfcore/sareksnpeff:2.7.GRCh38 +$ singularity pull --name nfcore-sarekvep-2.7.GRCh38.img docker://nfcore/sarekvep:2.7.GRCh38 + +# Move specific Singularity images into nf-core download folder +$ mv *.img nf-core-sarek-2.7/singularity-images/. + +# Connect to wharf using sftp +$ sftp -@bianca-sftp.uppmax.uu.se:- + +# Transfer rnaseq folder from rackham to wharf +sftp> put -r nf-core-rnaseq-3.0 . +[...] + +# Transfer sarek folder from rackham to wharf +sftp> put -r nf-core-sarek-2.7 . +[...] + +# The archives will be in the wharf folder in your user home on your bianca project + +# Connect to bianca +$ ssh -A -@bianca.uppmax.uu.se + +# Go to your project +$ cd /castor/project/proj_nobackup + +# Make and go into a nf-core/sarek directory (where you will store all nf-core/sarek versions) +$ mkdir sarek +$ cd sarek + +# Copy the tar from wharf to the project +$ cp /castor/project/proj_nobackup/wharf//-/nf-core-sarek-2.7 /castor/project/proj_nobackup/sarek + +# If you want other people to use it, +# Be sure that your group has rights to the directory as well +$ chown -R . nf-core-sarek-2.7 + +# Make a symbolic link to the extracted repository +$ ln -s nf-core-sarek-2.7 default +``` + +The principle is to have every member of your project to be able to use the same nf-core/sarek version at the same time. So every member of the project who wants to use nf-core/sarek will need to do: + +```bash +# Connect to bianca +$ ssh -A -@bianca.uppmax.uu.se + +# Go to your user directory +$ cd /home/ + +# Make a symbolic link to the default nf-core/sarek +$ ln -s /castor/project/proj_nobackup/sarek/default sarek +``` + +And then nf-core/sarek can be used with: + +```bash +$ nextflow run ~/sarek -profile uppmax --project --genome [GENOME ASSEMBLY] ... +``` + +This is an example of how to run sarek with the tool Manta and the genome assembly version GRCh38, which worked on `bianca` 20210416 + +```bash +$ nextflow run ~/sarek -profile uppmax --project --tools Manta --input +``` + +## Update nf-core/sarek + +Repeat the same steps as for installing nf-core/sarek, and update the link. + +```bash +# Connect to bianca (Connect to rackham first if needed) +$ ssh -A -@bianca.uppmax.uu.se + +# Go to the sarek directory in your project +$ cd /castor/project/proj_nobackup/sarek + +# Remove link +$ rm default + +# Link to new nf-core/sarek version +$ ln -s nf-core-sarek-2.7 default +``` + +You can for example keep a `default` version that you are sure is working, an make a link for a `testing` or `development` From 4930795cabe69b5ceb3e6c86f7817d28e639eb6b Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 20 Apr 2021 15:38:49 +0200 Subject: [PATCH 65/86] feat: more complete docs --- docs/uppmax.md | 178 ++++++++++++++++++++++++------------------------- 1 file changed, 88 insertions(+), 90 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index c8375a2..98b7d27 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -8,30 +8,32 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ ## Using the UPPMAX config profile -To use, run the pipeline with `-profile uppmax` (one hyphen). -This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. -Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - -In addition to this config profile, you will also need to specify an UPPMAX project id. -You can do this with the `--project` flag (two hyphens) when launching nextflow. For example: - -```bash -nextflow run nf-core/PIPELINE -profile uppmax --project snic2018-1-234 # ..rest of pipeline flags -``` - -$ NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. - Before running the pipeline you will need to either install Nextflow or load it using the environment module system. -This config enables Nextflow to manage the pipeline jobs via the Slurm job scheduler and using Singularity for software management. +To use, run the pipeline with `-profile uppmax` (one hyphen). +This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. +It will enable Nextflow to manage the pipeline jobs via the Slurm job scheduler. +Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) before execution of the pipeline. + +In addition to this config profile, you will also need to specify an UPPMAX project id. +You can do this with the `--project` flag (two hyphens) when launching nextflow. +For example: + +```bash +# Launch a nf-core pipeline with the uppmax profile for the project id snic2018-1-234 +$ nextflow run nf-core/ -profile uppmax --project snic2018-1-234 [...] +``` + +> NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. Just run Nextflow on a login node and it will handle everything else. Remember to use `-bg` to launch Nextflow in the background, so that the pipeline doesn't exit if you leave your terminal session. +Alternatively, you can also launch Nextflow in a `screen` or a `tmux` session. ## Using iGenomes references -A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `igenomes.config`. +A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `conf/igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Getting more memory @@ -43,7 +45,7 @@ If your nf-core pipeline run is running out of memory, you can run on a fat node ``` This raises the ceiling of available memory from the default of `128.GB` to `256.GB`. -`Rackham` has nodes with 128GB, 256GB and 1TB memory available. +`rackham` has nodes with 128GB, 256GB and 1TB memory available. Note that each job will still start with the same request as normal, but restarted attempts with larger requests will be able to request greater amounts of memory. @@ -53,15 +55,15 @@ All jobs will be submitted to fat nodes using this method, so it's only for use The UPPMAX nf-core configuration profile uses the `hostname` of the active environment to automatically apply the following resource limits: +* `rackham` + * cpus available: 20 cpus + * memory available: 125 GB * `bianca` * cpus available: 16 cpus * memory available: 109 GB * `irma` * cpus available: 16 cpus * memory available: 250 GB -* `rackham` - * cpus available: 20 cpus - * memory available: 125 GB ## Development config @@ -74,42 +76,41 @@ It is not suitable for use with real data. To use it, submit with `-profile uppmax,devel`. -## Running on Bianca +## Running on bianca -$ :warning: For more information about `bianca`, follow the [UPPMAX `bianca` user guide](http://uppmax.uu.se/support/user-guides/bianca-user-guide/). -$ :warning: For more information, follow the [nf-core guide for running offline](https://nf-co.re/usage/offline) and the [nf-core `tools` guide for downloading pipelines for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). - > :warning: For more information about using `Singularity` with UPPMAX, follow the [UPPMAX `Singularity` guide](https://www.uppmax.uu.se/support-sv/user-guides/singularity-user-guide/). +> :warning: For more information about `bianca`, follow the [UPPMAX `bianca` user guide](http://uppmax.uu.se/support/user-guides/bianca-user-guide/). +> :warning: For more information, follow the [nf-core guide for running offline](https://nf-co.re/usage/offline) and the [nf-core `tools` guide for downloading pipelines for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). +> :warning: For more information about using `Singularity` with UPPMAX, follow the [UPPMAX `Singularity` guide](https://www.uppmax.uu.se/support-sv/user-guides/singularity-user-guide/). For security reasons, there is no internet access on `bianca` so you can't download from or upload files to the cluster directly. -Before running a nf-core pipeline on `bianca` you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the wharf area to your own `bianca` project. +Before running a nf-core pipeline on `bianca` you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the `wharf` area to your own `bianca` project. -In this guide, we use `rackham` to download and transfer files to wharf, but it can also be done on your own computer. +In this guide, we use `rackham` to download and transfer files to the `wharf` area, but it can also be done on your own computer. If you use `rackham` to download the pipeline and the singularity containers, we recommend using an interactive session (cf [interactive guide](https://www.uppmax.uu.se/support/faq/running-jobs-faq/how-can-i-run-interactively-on-a-compute-node/)), which is what we do in the following guide. -### Download Nextflow +### Download and install Nextflow -You can use the UPPMAX provided `module`, but if necessary, you can also download a more recent version. +You can use the Nextflow UPPMAX provided `module`, but if necessary, you can also download a more recent version. ```bash # Connect to rackham -$ ssh -X @rackham.uppmax.uu.se +$ ssh -X @rackham.uppmax.uu.se # Or stay in your terminal # Download the nextflow-all bundle -$ wget https://github.com/nextflow-io/nextflow/releases/download/v/nextflow--all +$ wget https://github.com/nextflow-io/nextflow/releases/download/v/nextflow--all -# Connect to wharf using sftp -# For FileZilla follow the bianca user guide -$ sftp -@bianca-sftp.uppmax.uu.se:- +# Connect to the wharf area using sftp +$ sftp -@bianca-sftp.uppmax.uu.se:- -# Transfer nextflow to wharf -sftp> put nextflow--all . +# Transfer nextflow to the wharf area +sftp> put nextflow--all . # Exit sftp $ exit # Connect to bianca -$ ssh -A -@bianca.uppmax.uu.se +$ ssh -A -@bianca.uppmax.uu.se # Go to your project $ cd /castor/project/proj_nobackup @@ -118,19 +119,18 @@ $ cd /castor/project/proj_nobackup $ mkdir tools $ mkdir tools/nextflow -# Move Nextflow from wharf to its directory -$ mv /castor/project/proj_nobackup/wharf//-/nextflow--all /castor/project/proj_nobackup/tools/nextflow +# Move Nextflow from the wharf area to its directory +$ mv /castor/project/proj_nobackup/wharf//-/nextflow--all /castor/project/proj_nobackup/tools/nextflow # Establish permission -$ chmod a+x /castor/project/proj_nobackup/tools/nextflow/nextflow--all +$ chmod a+x /castor/project/proj_nobackup/tools/nextflow/nextflow--all # If you want other people to use it # Be sure that your group has rights to the directory as well - -$ chown -R . /castor/project/proj_nobackup/tools/nextflow/nextflow--all +$ chown -R . /castor/project/proj_nobackup/tools/nextflow/nextflow--all # Make a link to it -$ ln -s /castor/project/proj_nobackup/tools/nextflow/nextflow--all /castor/project/proj_nobackup/tools/nextflow/nextflow +$ ln -s /castor/project/proj_nobackup/tools/nextflow/nextflow--all /castor/project/proj_nobackup/tools/nextflow/nextflow # And every time you're launching Nextflow, don't forget to export the following ENV variables # Or add them to your .bashrc file @@ -141,107 +141,105 @@ $ export NXF_LAUNCHER=$SNIC_TMP $ export NXF_SINGULARITY_CACHEDIR=/castor/project/proj_nobackup/singularity-images ``` -### Download nf-core pipelines +### Install nf-core tools ```bash # Connect to rackham -$ ssh -X @rackham.uppmax.uu.se +$ ssh -X @rackham.uppmax.uu.se +# Or stay in your terminal + +# Install the latest pip version +$ pip3 install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev --user +``` + +### Download and transfer a nf-core pipeline + +```bash +# Connect to rackham +$ ssh -X @rackham.uppmax.uu.se # Or stay in your terminal # Open an interactive session (if you are on rackham) $ interactive -# Install the latest pip version -$ pip3 install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev --user - # Download a pipeline (nf-core/rnaseq 3.0) with the singularity images -$ nf-core download rnaseq -r 3.0 -s -p 10 --compress none +$ nf-core download rnaseq -r 3.0 -s --compress none -# Download a pipeline (nf-core/sarek 2.7) with the singularity images -$ nf-core download sarek -r 2.7 -s -p 10 --compress none - -# Download specific Singularity images +# If necessary, extra singularity images can be download separately +# For example, if you downloaded nf-core/sarek, you will need extra images for annotation +# Here we download the nf-core/sarek GRCh38 specific images $ singularity pull --name nfcore-sareksnpeff-2.7.GRCh38.img docker://nfcore/sareksnpeff:2.7.GRCh38 $ singularity pull --name nfcore-sarekvep-2.7.GRCh38.img docker://nfcore/sarekvep:2.7.GRCh38 -# Move specific Singularity images into nf-core download folder +# Which can then be moved into the nf-core/sarek download folder $ mv *.img nf-core-sarek-2.7/singularity-images/. -# Connect to wharf using sftp -$ sftp -@bianca-sftp.uppmax.uu.se:- +# Connect to the wharf area using sftp +$ sftp -@bianca-sftp.uppmax.uu.se:- -# Transfer rnaseq folder from rackham to wharf +# Transfer rnaseq folder from rackham to the wharf area sftp> put -r nf-core-rnaseq-3.0 . -[...] - -# Transfer sarek folder from rackham to wharf -sftp> put -r nf-core-sarek-2.7 . -[...] # The archives will be in the wharf folder in your user home on your bianca project # Connect to bianca -$ ssh -A -@bianca.uppmax.uu.se +$ ssh -A -@bianca.uppmax.uu.se # Go to your project $ cd /castor/project/proj_nobackup -# Make and go into a nf-core/sarek directory (where you will store all nf-core/sarek versions) -$ mkdir sarek -$ cd sarek +# Make and go into a nf-core directory (where you will store all nf-core pipelines') +$ mkdir f-core +$ cd nf-core -# Copy the tar from wharf to the project -$ cp /castor/project/proj_nobackup/wharf//-/nf-core-sarek-2.7 /castor/project/proj_nobackup/sarek +# Move the folder from the wharf area to the project +$ cp /castor/project/proj_nobackup/wharf//-/nf-core-rnaseq-3.0 . # If you want other people to use it, # Be sure that your group has rights to the directory as well -$ chown -R . nf-core-sarek-2.7 +$ chown -R . nf-core-rnaseq-3.0 # Make a symbolic link to the extracted repository -$ ln -s nf-core-sarek-2.7 default +$ ln -s nf-core-rnaseq-3.0 nf-core-rnaseq-default ``` -The principle is to have every member of your project to be able to use the same nf-core/sarek version at the same time. So every member of the project who wants to use nf-core/sarek will need to do: +The principle is to have every member of your project to be able to use the same nf-core/rnaseq version at the same time. +So every member of the project who wants to use nf-core/rnaseq will need to do: ```bash # Connect to bianca -$ ssh -A -@bianca.uppmax.uu.se +$ ssh -A -@bianca.uppmax.uu.se # Go to your user directory -$ cd /home/ +$ cd /home/ -# Make a symbolic link to the default nf-core/sarek -$ ln -s /castor/project/proj_nobackup/sarek/default sarek +# Make a symbolic link to the default nf-core/rnaseq +$ ln -s /castor/project/proj_nobackup/nf-core/nf-core-rnaseq-default nf-core-rnaseq ``` -And then nf-core/sarek can be used with: +And then nf-core/rnaseq can be used with: ```bash -$ nextflow run ~/sarek -profile uppmax --project --genome [GENOME ASSEMBLY] ... +# run rnaseq on bianca +$ nextflow run ~/rnaseq -profile uppmax --project --genome [GENOME ASSEMBLY] ... ``` -This is an example of how to run sarek with the tool Manta and the genome assembly version GRCh38, which worked on `bianca` 20210416 +## Update a pipeline -```bash -$ nextflow run ~/sarek -profile uppmax --project --tools Manta --input -``` - -## Update nf-core/sarek - -Repeat the same steps as for installing nf-core/sarek, and update the link. +To update, repeat the same steps as for installing and update the link. ```bash # Connect to bianca (Connect to rackham first if needed) -$ ssh -A -@bianca.uppmax.uu.se +$ ssh -A -@bianca.uppmax.uu.se -# Go to the sarek directory in your project -$ cd /castor/project/proj_nobackup/sarek +# Go to the rnaseq directory in your project +$ cd /castor/project/proj_nobackup/nf-core # Remove link -$ rm default +$ unlink nf-core-rnaseq-default -# Link to new nf-core/sarek version -$ ln -s nf-core-sarek-2.7 default +# Link to new nf-core/rnaseq version +$ ln -s nf-core-rnaseq-3.0 nf-core-rnaseq-default ``` -You can for example keep a `default` version that you are sure is working, an make a link for a `testing` or `development` +You can for example keep a `nf-core-rnaseq-default` version that you are sure is working, an make a link for a `nf-core-rnaseq-testing` or `nf-core-rnaseq-development`. From e4d2ef0984e049fb5cd9e539de5f833654987c08 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 20 Apr 2021 15:53:54 +0200 Subject: [PATCH 66/86] Apply suggestions from code review Co-authored-by: Phil Ewels --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 98b7d27..4097541 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -189,7 +189,7 @@ $ ssh -A -@bianca.uppmax.uu.se $ cd /castor/project/proj_nobackup # Make and go into a nf-core directory (where you will store all nf-core pipelines') -$ mkdir f-core +$ mkdir nf-core $ cd nf-core # Move the folder from the wharf area to the project From 1c1dd952fc28f59eec2c5e7724c5740c987c3469 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 20 Apr 2021 15:58:51 +0200 Subject: [PATCH 67/86] fix: bullet list --- docs/uppmax.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 4097541..c17063c 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -78,9 +78,12 @@ To use it, submit with `-profile uppmax,devel`. ## Running on bianca -> :warning: For more information about `bianca`, follow the [UPPMAX `bianca` user guide](http://uppmax.uu.se/support/user-guides/bianca-user-guide/). -> :warning: For more information, follow the [nf-core guide for running offline](https://nf-co.re/usage/offline) and the [nf-core `tools` guide for downloading pipelines for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). -> :warning: For more information about using `Singularity` with UPPMAX, follow the [UPPMAX `Singularity` guide](https://www.uppmax.uu.se/support-sv/user-guides/singularity-user-guide/). +> :warning: For more information, please follow the following guides: +> +> * [UPPMAX `bianca` user guide](http://uppmax.uu.se/support/user-guides/bianca-user-guide/). +> * [nf-core guide for running offline](https://nf-co.re/usage/offline) +> * [nf-core `tools` guide for downloading pipelines for offline use](https://nf-co.re/tools#downloading-pipelines-for-offline-use). +> * [UPPMAX `Singularity` guide](https://www.uppmax.uu.se/support-sv/user-guides/singularity-user-guide/). For security reasons, there is no internet access on `bianca` so you can't download from or upload files to the cluster directly. Before running a nf-core pipeline on `bianca` you will first have to download the pipeline and singularity images needed elsewhere and transfer them via the `wharf` area to your own `bianca` project. From 4eea06f0dbeb9189496ef6881bffc95bc8b19935 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 20 Apr 2021 16:03:34 +0200 Subject: [PATCH 68/86] Apply suggestions from code review Co-authored-by: Pontus Freyhult --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index c17063c..621448b 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -8,7 +8,7 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ ## Using the UPPMAX config profile -Before running the pipeline you will need to either install Nextflow or load it using the environment module system. +Before running the pipeline you will need to either install Nextflow or load it using the environment module system (this can be done with e.g. `module load bioinfo-tools Nextflow/` where `VERSION` is e.g. `20.10`). To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. From b225a0555ae00df959ac43afe946f02e24a2a8fe Mon Sep 17 00:00:00 2001 From: phue Date: Wed, 21 Apr 2021 10:08:00 +0200 Subject: [PATCH 69/86] biohpc_gen: update software tree path --- conf/biohpc_gen.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/biohpc_gen.config b/conf/biohpc_gen.config index b625b20..e3f4069 100755 --- a/conf/biohpc_gen.config +++ b/conf/biohpc_gen.config @@ -12,7 +12,7 @@ env { process { executor = 'slurm' queue = { task.memory <= 1536.GB ? (task.time > 2.d || task.memory > 384.GB ? 'biohpc_gen_production' : 'biohpc_gen_normal') : 'biohpc_gen_highmem' } - beforeScript = 'module use /dss/dssfs01/pr53da/pr53da-dss-0000/spack/modules/x86_avx2/linux*' + beforeScript = 'module use /dss/dsslegfs02/pn73se/pn73se-dss-0000/spack/modules/x86_avx2/linux*' module = 'charliecloud/0.22:miniconda3' } From 79e3dd65a753f69b17b9f7f4c043b73ed4030619 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 21 Apr 2021 11:06:39 +0200 Subject: [PATCH 70/86] Apply suggestions from code review Co-authored-by: Pontus Freyhult --- docs/uppmax.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 621448b..1cf8690 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -13,7 +13,9 @@ Before running the pipeline you will need to either install Nextflow or load it To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. It will enable Nextflow to manage the pipeline jobs via the Slurm job scheduler. -Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) before execution of the pipeline. +Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) if needed before execution of the pipeline. + +Recent version of Nextflow also support the environment variable `NXF_SINGULARITY_CACHEDIR` which can be used to supply images. Images for some nf-core pipelines are available under `/sw/data/ToolBox/nf-core/` and those can be used by `NXF_SINGULARITY_CACHEDIR=/sw/data/ToolBox/nf-core/; export NXF_SINGULARITY_CACHEDIR`. In addition to this config profile, you will also need to specify an UPPMAX project id. You can do this with the `--project` flag (two hyphens) when launching nextflow. From 658fa7ccda13d2d5d9c9d220be7524ae4463f7a4 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 21 Apr 2021 11:08:16 +0200 Subject: [PATCH 71/86] fix: markdownlint --- docs/uppmax.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 1cf8690..9057b2a 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -13,9 +13,10 @@ Before running the pipeline you will need to either install Nextflow or load it To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. It will enable Nextflow to manage the pipeline jobs via the Slurm job scheduler. -Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) if needed before execution of the pipeline. +Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) if needed before execution of the pipeline. -Recent version of Nextflow also support the environment variable `NXF_SINGULARITY_CACHEDIR` which can be used to supply images. Images for some nf-core pipelines are available under `/sw/data/ToolBox/nf-core/` and those can be used by `NXF_SINGULARITY_CACHEDIR=/sw/data/ToolBox/nf-core/; export NXF_SINGULARITY_CACHEDIR`. +Recent version of Nextflow also support the environment variable `NXF_SINGULARITY_CACHEDIR` which can be used to supply images. +Images for some nf-core pipelines are available under `/sw/data/ToolBox/nf-core/` and those can be used by `NXF_SINGULARITY_CACHEDIR=/sw/data/ToolBox/nf-core/; export NXF_SINGULARITY_CACHEDIR`. In addition to this config profile, you will also need to specify an UPPMAX project id. You can do this with the `--project` flag (two hyphens) when launching nextflow. From c89877decca23ea327a50bd9ec5665fb8e9675fe Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 21 Apr 2021 11:17:06 +0200 Subject: [PATCH 72/86] feat: use --- docs/uppmax.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index 9057b2a..a38c1be 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -168,8 +168,8 @@ $ ssh -X @rackham.uppmax.uu.se # Open an interactive session (if you are on rackham) $ interactive -# Download a pipeline (nf-core/rnaseq 3.0) with the singularity images -$ nf-core download rnaseq -r 3.0 -s --compress none +# Download a pipeline with the singularity images +$ nf-core download -r -s --compress none # If necessary, extra singularity images can be download separately # For example, if you downloaded nf-core/sarek, you will need extra images for annotation @@ -183,8 +183,8 @@ $ mv *.img nf-core-sarek-2.7/singularity-images/. # Connect to the wharf area using sftp $ sftp -@bianca-sftp.uppmax.uu.se:- -# Transfer rnaseq folder from rackham to the wharf area -sftp> put -r nf-core-rnaseq-3.0 . +# Transfer folder from rackham to the wharf area +sftp> put -r nf-core-- . # The archives will be in the wharf folder in your user home on your bianca project @@ -199,18 +199,18 @@ $ mkdir nf-core $ cd nf-core # Move the folder from the wharf area to the project -$ cp /castor/project/proj_nobackup/wharf//-/nf-core-rnaseq-3.0 . +$ cp /castor/project/proj_nobackup/wharf//-/nf-core-- . # If you want other people to use it, # Be sure that your group has rights to the directory as well -$ chown -R . nf-core-rnaseq-3.0 +$ chown -R . nf-core-- # Make a symbolic link to the extracted repository -$ ln -s nf-core-rnaseq-3.0 nf-core-rnaseq-default +$ ln -s nf-core-- nf-core--default ``` -The principle is to have every member of your project to be able to use the same nf-core/rnaseq version at the same time. -So every member of the project who wants to use nf-core/rnaseq will need to do: +The principle is to have every member of your project to be able to use the same nf-core/ version at the same time. +So every member of the project who wants to use nf-core/ will need to do: ```bash # Connect to bianca @@ -219,15 +219,15 @@ $ ssh -A -@bianca.uppmax.uu.se # Go to your user directory $ cd /home/ -# Make a symbolic link to the default nf-core/rnaseq -$ ln -s /castor/project/proj_nobackup/nf-core/nf-core-rnaseq-default nf-core-rnaseq +# Make a symbolic link to the default nf-core/ +$ ln -s /castor/project/proj_nobackup/nf-core/nf-core--default nf-core- ``` -And then nf-core/rnaseq can be used with: +And then nf-core/ can be used with: ```bash -# run rnaseq on bianca -$ nextflow run ~/rnaseq -profile uppmax --project --genome [GENOME ASSEMBLY] ... +# run on bianca +$ nextflow run ~/ -profile uppmax --project --genome ... ``` ## Update a pipeline @@ -238,14 +238,14 @@ To update, repeat the same steps as for installing and update the link. # Connect to bianca (Connect to rackham first if needed) $ ssh -A -@bianca.uppmax.uu.se -# Go to the rnaseq directory in your project +# Go to the nf-core directory in your project $ cd /castor/project/proj_nobackup/nf-core # Remove link -$ unlink nf-core-rnaseq-default +$ unlink nf-core--default -# Link to new nf-core/rnaseq version -$ ln -s nf-core-rnaseq-3.0 nf-core-rnaseq-default +# Link to new nf-core/ version +$ ln -s nf-core-- nf-core--default ``` -You can for example keep a `nf-core-rnaseq-default` version that you are sure is working, an make a link for a `nf-core-rnaseq-testing` or `nf-core-rnaseq-development`. +You can for example keep a `nf-core--default` version that you are sure is working, an make a link for a `nf-core--testing` or `nf-core--development`. From 1a7ec56c33adc7550ccff48b2ffecc5e638e4a8e Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 21 Apr 2021 11:23:50 +0200 Subject: [PATCH 73/86] feat: more details --- docs/uppmax.md | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index a38c1be..176adf3 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -8,18 +8,18 @@ We have a Slack channel dedicated to UPPMAX users on the nf-core Slack: [https:/ ## Using the UPPMAX config profile -Before running the pipeline you will need to either install Nextflow or load it using the environment module system (this can be done with e.g. `module load bioinfo-tools Nextflow/` where `VERSION` is e.g. `20.10`). +Before running the pipeline you will need to either install `Nextflow` or load it using the environment module system (this can be done with e.g. `module load bioinfo-tools Nextflow/` where `VERSION` is e.g. `20.10`). To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. -It will enable Nextflow to manage the pipeline jobs via the Slurm job scheduler. -Using this profile, Docker image(s) containing required software(s) will be downloaded, and converted to Singularity image(s) if needed before execution of the pipeline. +It will enable `Nextflow` to manage the pipeline jobs via the `Slurm` job scheduler. +Using this profile, `Docker` image(s) containing required software(s) will be downloaded, and converted to `Singularity` image(s) if needed before execution of the pipeline. -Recent version of Nextflow also support the environment variable `NXF_SINGULARITY_CACHEDIR` which can be used to supply images. -Images for some nf-core pipelines are available under `/sw/data/ToolBox/nf-core/` and those can be used by `NXF_SINGULARITY_CACHEDIR=/sw/data/ToolBox/nf-core/; export NXF_SINGULARITY_CACHEDIR`. +Recent version of `Nextflow` also support the environment variable `NXF_SINGULARITY_CACHEDIR` which can be used to supply images. +Images for some `nf-core` pipelines are available under `/sw/data/ToolBox/nf-core/` and those can be used by `NXF_SINGULARITY_CACHEDIR=/sw/data/ToolBox/nf-core/; export NXF_SINGULARITY_CACHEDIR`. In addition to this config profile, you will also need to specify an UPPMAX project id. -You can do this with the `--project` flag (two hyphens) when launching nextflow. +You can do this with the `--project` flag (two hyphens) when launching `Nextflow`. For example: ```bash @@ -29,19 +29,19 @@ $ nextflow run nf-core/ -profile uppmax --project snic2018-1-234 [...] > NB: If you're not sure what your UPPMAX project ID is, try running `groups` or checking SUPR. -Just run Nextflow on a login node and it will handle everything else. +Just run `Nextflow` on a login node and it will handle everything else. -Remember to use `-bg` to launch Nextflow in the background, so that the pipeline doesn't exit if you leave your terminal session. -Alternatively, you can also launch Nextflow in a `screen` or a `tmux` session. +Remember to use `-bg` to launch `Nextflow` in the background, so that the pipeline doesn't exit if you leave your terminal session. +Alternatively, you can also launch `Nextflow` in a `screen` or a `tmux` session. -## Using iGenomes references +## Using AWS iGenomes references -A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `conf/igenomes.config`. +A local copy of the `AWS iGenomes` resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `conf/igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Getting more memory -If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: +If your `nf-core` pipeline run is running out of memory, you can run on a fat node with more memory using the following `Nextflow` flags: ```bash --clusterOptions "-C mem256GB" --max_memory "256GB" @@ -96,7 +96,15 @@ If you use `rackham` to download the pipeline and the singularity containers, we ### Download and install Nextflow -You can use the Nextflow UPPMAX provided `module`, but if necessary, you can also download a more recent version. +You can use the `Nextflow` UPPMAX provided `module`, but if necessary, you can also download a more recent version. + +```bash +# See the available versions for the module +module spider Nextflow + +# Load a specific version of the Nextflow module +module load bioinfo-tools Nextflow/` +``` ```bash # Connect to rackham @@ -149,6 +157,19 @@ $ export NXF_SINGULARITY_CACHEDIR=/castor/project/proj_nobackup/singularity-imag ### Install nf-core tools +You can use the `nf-core` UPPMAX provided `module`, but if necessary, you can also download a more recent version. + +```bash +# Connect to rackham +$ ssh -X @rackham.uppmax.uu.se + +# See the available versions for the module +module spider nf-core + +# Load a specific version of the nf-core module +module load bioinfo-tools nf-core/` +``` + ```bash # Connect to rackham $ ssh -X @rackham.uppmax.uu.se From 844246cc7d649c271d7d72467b8196ebc3b24a0e Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 21 Apr 2021 11:28:58 +0200 Subject: [PATCH 74/86] fix: clearer usage of module --- docs/uppmax.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/uppmax.md b/docs/uppmax.md index 176adf3..e8304b8 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -99,6 +99,9 @@ If you use `rackham` to download the pipeline and the singularity containers, we You can use the `Nextflow` UPPMAX provided `module`, but if necessary, you can also download a more recent version. ```bash +# Connect to bianca +$ ssh -A -@bianca.uppmax.uu.se + # See the available versions for the module module spider Nextflow From bab9241243ed2a92a99f31fe9ee9cb39d9b99405 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 21 Apr 2021 11:51:14 +0200 Subject: [PATCH 75/86] fix: markdownlint --- docs/uppmax.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index e8304b8..a26fa88 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -233,8 +233,8 @@ $ chown -R . nf-core-- $ ln -s nf-core-- nf-core--default ``` -The principle is to have every member of your project to be able to use the same nf-core/ version at the same time. -So every member of the project who wants to use nf-core/ will need to do: +The principle is to have every member of your project to be able to use the same `nf-core/` version at the same time. +So every member of the project who wants to use `nf-core/` will need to do: ```bash # Connect to bianca @@ -247,7 +247,7 @@ $ cd /home/ $ ln -s /castor/project/proj_nobackup/nf-core/nf-core--default nf-core- ``` -And then nf-core/ can be used with: +And then `nf-core/` can be used with: ```bash # run on bianca From 1548690ca68b8c7501352a7783e8ca649e2b00bf Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 21 Apr 2021 12:27:27 +0200 Subject: [PATCH 76/86] Update docs/uppmax.md Co-authored-by: Pontus Freyhult --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index a26fa88..3024cb0 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -44,7 +44,7 @@ You can do this by simply using the `--genome ` parameter. If your `nf-core` pipeline run is running out of memory, you can run on a fat node with more memory using the following `Nextflow` flags: ```bash ---clusterOptions "-C mem256GB" --max_memory "256GB" +--clusterOptions "-C mem256GB -p node" --max_memory "256GB" ``` This raises the ceiling of available memory from the default of `128.GB` to `256.GB`. From 7d8770af1a1ad08fdaa9d83bde33e1e541969de2 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 16:39:55 +0100 Subject: [PATCH 77/86] Create sanger.config --- conf/sanger.config | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 conf/sanger.config diff --git a/conf/sanger.config b/conf/sanger.config new file mode 100644 index 0000000..686a3af --- /dev/null +++ b/conf/sanger.config @@ -0,0 +1,26 @@ +params { + config_profile_description = 'The Wellcome Sanger Institute HPC cluster profile' + config_profile_contact = 'Anthony Underwood (@aunderwo)' + config_profile_url = 'https://www.sanger.ac.uk/group/informatics-support-group/' +} + +singularity { + enabled = true + cacheDir = "${baseDir}/singularity" + runOptions = '--bind /lustre --bind /nfs/pathnfs01 --bind /nfs/pathnfs02 --bind /nfs/pathnfs03 --bind /nfs/pathnfs04 --bind /nfs/pathnfs05 --bind /nfs/pathnfs06 --no-home' +} + +process{ + executor = 'lsf' + queue = 'normal' + errorStrategy = { task.attempt <= 5 ? "retry" : "finish" } + process.maxRetries = 5 +} + +executor{ + name = 'lsf' + perJobMemLimit = true + poolSize = 4 + submitRateLimit = '5 sec' + killBatchSize = 50 +} From 7163e8723d7f7011bfb3f6aa225c170848a6c5c1 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:17:42 +0100 Subject: [PATCH 78/86] Create sanger.md --- docs/sanger.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 docs/sanger.md diff --git a/docs/sanger.md b/docs/sanger.md new file mode 100644 index 0000000..be39bfa --- /dev/null +++ b/docs/sanger.md @@ -0,0 +1,47 @@ +# nf-core/configs: Wellcome Sanger Institute Configuration + +To use, run the pipeline with `-profile sanger`. This will download and launch the [`sanger.config`](../conf/sanger.config) which has been +pre-configured with a setup suitable for the Wellcome Sanger Institute LSF cluster. +Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or +a Singularity image downloaded directly before execution of the pipeline. + +## Running the workflow on the Wellcome Sanger Institute cluster + +The latest version of Nextflow is not installed by default on the cluster. You will need to install it into a directory you have write access to + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. +Nextflow shouldn't run directly on the submission node but on a compute node. +To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` + +``` +#!/bin/bash +#BSUB -o /path/to/a/log/dir/%J.o +#BSUB -e /path/to/a/log/dir//%J.e +#BSUB -M 8000 +#BSUB -q long +#BSUB -n 4 + +export HTTP_PROXY='http://wwwcache.sanger.ac.uk:3128' +export HTTPS_PROXY='http://wwwcache.sanger.ac.uk:3128' +export NXF_ANSI_LOG=false +export NXF_OPTS="-Xms8G -Xmx8G -Dnxf.pool.maxThreads=2000" +export NXF_VER=21.03.0-edge + + +/path/to/your/install/nextflow run \ +/path/to/nf-core/pipeline/main.nf \ +-w /path/to/some/dir/work \ +-profile sanger \ +-c my_specific.config \ +-qs 1000 \ +-resume + +## clean up on exit 0 - delete this if you want to keep the work dir +status=$? +if [[ $status -eq 0 ]]; then + rm -r /path/to/some/dir/work +fi +``` + From 40eb3e02010af4653b44748116102eb6f6b0f376 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:20:30 +0100 Subject: [PATCH 79/86] Update sanger.md --- docs/sanger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index be39bfa..f1e5c29 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -2,7 +2,7 @@ To use, run the pipeline with `-profile sanger`. This will download and launch the [`sanger.config`](../conf/sanger.config) which has been pre-configured with a setup suitable for the Wellcome Sanger Institute LSF cluster. -Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or +Using this profile, either a docker image containing all of the required software will be downloaded, and converted to a Singularity image or a Singularity image downloaded directly before execution of the pipeline. ## Running the workflow on the Wellcome Sanger Institute cluster From cc18d213dc44742ac579b6e7aa6498656e6c7313 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:24:08 +0100 Subject: [PATCH 80/86] Update sanger.md --- docs/sanger.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/sanger.md b/docs/sanger.md index f1e5c29..bb0e607 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -15,7 +15,7 @@ Nextflow manages each process as a separate job that is submitted to the cluster Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` -``` +```bash #!/bin/bash #BSUB -o /path/to/a/log/dir/%J.o #BSUB -e /path/to/a/log/dir//%J.e @@ -44,4 +44,3 @@ if [[ $status -eq 0 ]]; then rm -r /path/to/some/dir/work fi ``` - From 6175362049ab8a378df5464970b23c6f40b00c1c Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 20:47:48 +0100 Subject: [PATCH 81/86] Add sanger profile --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 88d7a62..9b4fbda 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -43,6 +43,7 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } + sanger { includeConfig "${params.custom_config_base}/conf/sanger.config"} seg_globe { includeConfig "${params.custom_config_base}/conf/seg_globe.config"} shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hpc { includeConfig "${params.custom_config_base}/conf/uct_hpc.config" } From fa267c0cf95a5657d33ce07892b073b8a0629d57 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Wed, 21 Apr 2021 21:01:25 +0100 Subject: [PATCH 82/86] Add sanger to tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd2170c..65cabe5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'biohpc_gen', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'eva', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'sanger', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 351c5773f0541dd445649e6c4af1b8a8590e0a5e Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 11:59:34 +0100 Subject: [PATCH 83/86] Add queue limits for 'normal' --- conf/sanger.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/sanger.config b/conf/sanger.config index 686a3af..63952c8 100644 --- a/conf/sanger.config +++ b/conf/sanger.config @@ -24,3 +24,9 @@ executor{ submitRateLimit = '5 sec' killBatchSize = 50 } + +params { + max_memory = 128.GB + max_cpus = 64 + max_time = 12.h +} From b16b011bcc2fc72ad7b3b84b410d895cae139cdf Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 12:02:11 +0100 Subject: [PATCH 84/86] Add ~/bin as recommended location for nextflow executable --- docs/sanger.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/sanger.md b/docs/sanger.md index bb0e607..483683c 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -11,6 +11,8 @@ The latest version of Nextflow is not installed by default on the cluster. You w - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) +Recommendation place to move the `nextflow` executable is `~/bin` so that it's in the `PATH`. + Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` @@ -27,10 +29,10 @@ export HTTP_PROXY='http://wwwcache.sanger.ac.uk:3128' export HTTPS_PROXY='http://wwwcache.sanger.ac.uk:3128' export NXF_ANSI_LOG=false export NXF_OPTS="-Xms8G -Xmx8G -Dnxf.pool.maxThreads=2000" -export NXF_VER=21.03.0-edge +export NXF_VER=21.04.0-edge -/path/to/your/install/nextflow run \ +nextflow run \ /path/to/nf-core/pipeline/main.nf \ -w /path/to/some/dir/work \ -profile sanger \ From f414446a8845852e4fd8032fb3d45da385e33f10 Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 15:38:36 +0100 Subject: [PATCH 85/86] Add info on how to load Singularity into your path --- docs/sanger.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index 483683c..940834e 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -11,9 +11,16 @@ The latest version of Nextflow is not installed by default on the cluster. You w - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) -Recommendation place to move the `nextflow` executable is `~/bin` so that it's in the `PATH`. +A recommended place to move the `nextflow` executable to is `~/bin` so that it's in the `PATH`. + Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. +Since the Nextflow pipeline will submit individual jobs for each process to the cluster and dependencies will be provided bu Singularity images you shoudl make sure that your account has access to the Singularity binary by adding these lines to your `.bashrc` file + +```bash +[[ -f /software/pathogen/farm5 ]] && module load ISG/singularity +``` + Nextflow shouldn't run directly on the submission node but on a compute node. To do so make a shell script with a similar structure to the following code and submit with `bsub < $PWD/my_script.sh` From 3e8f9b3d4e60f912039f10192a11ee099e5c668a Mon Sep 17 00:00:00 2001 From: Anthony Underwood Date: Thu, 22 Apr 2021 15:57:09 +0100 Subject: [PATCH 86/86] Remove blank lines --- docs/sanger.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sanger.md b/docs/sanger.md index 940834e..ee75755 100644 --- a/docs/sanger.md +++ b/docs/sanger.md @@ -13,7 +13,6 @@ The latest version of Nextflow is not installed by default on the cluster. You w A recommended place to move the `nextflow` executable to is `~/bin` so that it's in the `PATH`. - Nextflow manages each process as a separate job that is submitted to the cluster by using the `bsub` command. Since the Nextflow pipeline will submit individual jobs for each process to the cluster and dependencies will be provided bu Singularity images you shoudl make sure that your account has access to the Singularity binary by adding these lines to your `.bashrc` file