From 885f0ce5f69abafac3237728c929ba5b952eb593 Mon Sep 17 00:00:00 2001 From: Gisela Gabernet Garriga Date: Mon, 9 Nov 2020 22:41:38 +0100 Subject: [PATCH 01/44] awsbatch config update --- conf/awsbatch.config | 14 +++++++++++++- docs/awsbatch.md | 4 +++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 2e5f83c..43f90db 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -7,7 +7,19 @@ params { awsqueue = false awsregion = 'eu-west-1' awscli = '/home/ec2-user/miniconda/bin/aws' - tracedir = './' +} + +timeline { + overwrite = true +} +report { + overwrite = true +} +trace { + overwrite = true +} +dag { + overwrite = true } process.executor = 'awsbatch' diff --git a/docs/awsbatch.md b/docs/awsbatch.md index c00acef..daf425b 100644 --- a/docs/awsbatch.md +++ b/docs/awsbatch.md @@ -1,4 +1,6 @@ # nf-core/configs: awsbatch Configuration To be used with `awsbatch`. -Custom queue, region and CLI path can be supplied with `params.awsqueue`, `params.awsregion`, `params.awscli`, respectively. +Custom queue and region can be supplied with `params.awsqueue`, `params.awsregion`, `params.awscli`, respectively. + +Allow `overwrite` of `trace`, `timeline`, `report` and `dag` to allow resuming pipelines. From 9b5bcbd43436fd086d53790c73202343dbbc58bd Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 9 Feb 2021 22:06:59 +0100 Subject: [PATCH 02/44] update aws clipath --- conf/awsbatch.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 43f90db..a8b61b8 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -25,4 +25,4 @@ dag { process.executor = 'awsbatch' process.queue = params.awsqueue aws.region = params.awsregion -executor.awscli = params.awscli +aws.batch.cliPath = params.awscli From f283dfb2dcfec79b120953c9ee7d6e3ab4aec695 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 15 Feb 2021 16:58:30 +0000 Subject: [PATCH 03/44] Add ARTIC primer sets to genome config --- conf/pipeline/viralrecon/genomes.config | 34 +++++++++++++++++++++---- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index d889460..d321e57 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -4,17 +4,41 @@ * ------------------------------------------------- * Defines viral reference genomes for all environments. */ - + params { // Genome reference file paths genomes { 'NC_045512.2' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz" + // This version of the reference has been kept here for backwards compatibility. + // Please use 'MN908947.3' if possible because all primer sets are available / have been pre-prepared relative to that assembly + fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz' } 'MN908947.3' { - fasta = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz" - gff = "https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz" + fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz' + gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_sets { + artic { + '1' { + scheme = 'nCoV-2019' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' + fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.reference.fasta' + gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + } + '2' { + scheme = 'nCoV-2019' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' + fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.reference.fasta' + gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + } + '3' { + scheme = 'nCoV-2019' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' + fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.reference.fasta' + gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + } + } + } } } } From 9d500cf6790e593b91fa9e76d647e355a863009b Mon Sep 17 00:00:00 2001 From: drpatelh Date: Mon, 15 Feb 2021 17:02:26 +0000 Subject: [PATCH 04/44] Reorder params --- conf/pipeline/viralrecon/genomes.config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index d321e57..6321e8f 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -20,22 +20,22 @@ params { primer_sets { artic { '1' { - scheme = 'nCoV-2019' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' + scheme = 'nCoV-2019' } '2' { - scheme = 'nCoV-2019' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' + scheme = 'nCoV-2019' } '3' { - scheme = 'nCoV-2019' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' + scheme = 'nCoV-2019' } } } From 4de0855218d3c52fa38e0f72960216fc1947bdb9 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Mon, 15 Feb 2021 18:48:03 +0000 Subject: [PATCH 05/44] Update genomes.config --- conf/pipeline/viralrecon/genomes.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 6321e8f..07a470b 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -22,19 +22,19 @@ params { '1' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.scheme.bed' scheme = 'nCoV-2019' } '2' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.scheme.bed' scheme = 'nCoV-2019' } '3' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.scheme.bed' scheme = 'nCoV-2019' } } From 1f2fce00aaa541c5f262c0bcd20467bf32b92071 Mon Sep 17 00:00:00 2001 From: Marc Hoeppner Date: Tue, 16 Feb 2021 09:02:16 +0100 Subject: [PATCH 06/44] Updating the CCGA DX profile to new storage system --- conf/ccga_dx.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config index 1f92b0d..595d3e1 100644 --- a/conf/ccga_dx.config +++ b/conf/ccga_dx.config @@ -13,7 +13,7 @@ params { singularity { enabled = true - runOptions = "-B /mnt" + runOptions = "-B /mnt -B /work_ifs" } executor { @@ -30,7 +30,7 @@ process { params { // illumina iGenomes reference file paths on DX Cluster - igenomes_base = '/mnt/ld_ng_out/sukmb352/references/iGenomes/references/' + igenomes_base = '/work_ifs/ikmb_repository/references/iGenomes/references/' saveReference = true max_memory = 250.GB max_cpus = 20 From 6ffaf922c5d232f6c5bed94256bb731815cf9f82 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 16 Feb 2021 10:28:32 +0000 Subject: [PATCH 07/44] Update genomes.config --- conf/pipeline/viralrecon/genomes.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 07a470b..6321e8f 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -22,19 +22,19 @@ params { '1' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.scheme.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } '2' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.scheme.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } '3' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.reference.fasta' gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' - primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.scheme.bed' + primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } } From 01a4f8559e2e261b356a788a2948ec923733048b Mon Sep 17 00:00:00 2001 From: drpatelh Date: Tue, 16 Feb 2021 13:17:16 +0000 Subject: [PATCH 08/44] Update links to be consistent --- conf/pipeline/viralrecon/genomes.config | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 6321e8f..27f7162 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -11,29 +11,29 @@ params { 'NC_045512.2' { // This version of the reference has been kept here for backwards compatibility. // Please use 'MN908947.3' if possible because all primer sets are available / have been pre-prepared relative to that assembly - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' - gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz' + fasta = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.fna.gz' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/NC_045512.2/GCF_009858895.2_ASM985889v3_genomic.200409.gff.gz' } 'MN908947.3' { - fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz' - gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + fasta = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.fna.gz' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' primer_sets { artic { '1' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.reference.fasta' - gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V1/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } '2' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.reference.fasta' - gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V2/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } '3' { fasta = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.reference.fasta' - gff = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' primer_bed = 'https://github.com/artic-network/primer-schemes/raw/master/nCoV-2019/V3/nCoV-2019.primer.bed' scheme = 'nCoV-2019' } From 286e18b80c6e615406b96237a25b4e9d259d21cb Mon Sep 17 00:00:00 2001 From: maxibor Date: Wed, 17 Feb 2021 10:22:05 +0100 Subject: [PATCH 09/44] update shh config after CDAG departure --- conf/shh.config | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index ab3dcca..a17c935 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,10 +1,10 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Generic MPI-SHH cluster(s) profile provided by nf-core/configs.' + config_profile_description = 'Generic MPI-SHH SDAG cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' - max_memory = 256.GB - max_cpus = 32 + max_memory = 2.TB + max_cpus = 128 max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" @@ -13,14 +13,9 @@ params { // Preform work directory cleanup after a successful run cleanup = true -singularity { - enabled = true - autoMounts = true - cacheDir = "/projects1/singularity_scratch/cache/" -} - process { executor = 'slurm' + queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } executor { @@ -28,24 +23,10 @@ executor { } profiles { - cdag { - params { - config_profile_description = 'CDAG MPI-SHH profile, provided by nf-core/configs.' - } - // delete when CDAG will be fixed - process { - queue = 'long' - } - } - sdag { - params { - config_profile_description = 'SDAG MPI-SHH profile, provided by nf-core/configs.' - max_memory = 2.TB - max_cpus = 128 - } - process { - queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } - } + singularity { + singularity.enabled = true + singularity.autoMounts = true + singularity.cacheDir = "/projects1/singularity_scratch/cache/" } // Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option. debug { From 11e2083fb1ea0d38988ed6d68bf1b6051450237c Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 3 Mar 2021 09:57:34 +0000 Subject: [PATCH 10/44] Added eddie profile to nfcore_custom.config list --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 429e3c1..7abbb7e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,6 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } + eddie { includeConfig "${params.custom_config_bsae}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 3af2c6409f66f387c2b5aa7f039f435e0c74404c Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 3 Mar 2021 09:59:16 +0000 Subject: [PATCH 11/44] Initial commit of eddie.config --- conf/eddie.config | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 conf/eddie.config diff --git a/conf/eddie.config b/conf/eddie.config new file mode 100644 index 0000000..11ae1d9 --- /dev/null +++ b/conf/eddie.config @@ -0,0 +1,30 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'University of Edinburgh (eddie) cluster profile provided by nf-core/configs.' + config_profile_contact = 'Alison Meynert (@ameynert)' + config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' +} + +conda.createTimeout = '2h' + +process { + executor = 'sge' + + # memory environment & options + clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} + penv = 'sharedmem' + + # common SGE error statuses + errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} + maxErrors = '-1' + maxRetries = 3 + + # load module script for Anaconda + beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } + module = 'anaconda/5.3.1' +} +params { + saveReference = true + // illumina iGenomes reference file paths on GIS Aquila + igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' +} From 6f09a233f072e5878b5b8df803484d3ee3778f51 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 4 Mar 2021 14:46:23 +0000 Subject: [PATCH 12/44] Added singularity module --- conf/eddie.config | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 11ae1d9..5b156be 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -5,7 +5,13 @@ params { config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' } -conda.createTimeout = '2h' +conda { + createTimeout = '2h' +} + +singularity { + enabled = true +} process { executor = 'sge' @@ -19,12 +25,19 @@ process { maxErrors = '-1' maxRetries = 3 - # load module script for Anaconda + # load module script for Anaconda and Singularity beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } module = 'anaconda/5.3.1' + module = 'singularity/3.5.3' } + params { saveReference = true - // illumina iGenomes reference file paths on GIS Aquila + + // iGenomes reference base igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' + + max_memory = 384.GB + max_cpus = 32 + max_time = 240.h } From 4c0111f598935e7223c8d51f510a9271e959e41c Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 4 Mar 2021 14:46:58 +0000 Subject: [PATCH 13/44] Eddie profile documentation initial commit --- docs/eddie.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/eddie.md diff --git a/docs/eddie.md b/docs/eddie.md new file mode 100644 index 0000000..e6a2f87 --- /dev/null +++ b/docs/eddie.md @@ -0,0 +1,49 @@ +# nf-core/configs: Eddie Configuration + +nf-core pipelines sarek, rnaseq, and atacseq have all been tested on the University of Edinburgh Eddie HPC. + +## Getting help + +There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https://igmm.slack.com/channels/eddie3](https://igmm.slack.com/channels/eddie3) + +## Using the Eddie config profile + +To use, run the pipeline with `-profile eddie` (one hyphen). +This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). + +The configuration file supports running nf-core pipelines with either a Conda environment or Docker containers running under Singularity. + +```bash +nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +``` + +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. + +To list versions: + +```bash +module avail igmm/apps/nextflow +``` + +To load the most recent version: +```bash +module load igmm/apps/nextflow +``` + + +This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. + +To set up Nextflow on a login node ... TODO + +## Using iGenomes references + +A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. +You can do this by simply using the `--genome ` parameter. + +## Adjusting maximum resources + +This config is set for IGMM standard nodes which have 32 cores and 384GB memory. If you are a non-IGMM user, please see the [ECDF specification](https://www.wiki.ed.ac.uk/display/ResearchServices/Memory+Specification) and adjust the `--clusterOptions` flag appropriately, e.g. + +```bash +--clusterOptions "-C mem256GB" --max_memory "256GB" +``` From babeccd020011e62671b2df27c201a2cdb5cedae Mon Sep 17 00:00:00 2001 From: Graeme Grimes Date: Thu, 4 Mar 2021 16:45:21 +0000 Subject: [PATCH 14/44] changed # to // for comments --- conf/eddie.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 5b156be..3968783 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -16,16 +16,16 @@ singularity { process { executor = 'sge' - # memory environment & options + // memory environment & options clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} penv = 'sharedmem' - # common SGE error statuses + // common SGE error statuses errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} maxErrors = '-1' maxRetries = 3 - # load module script for Anaconda and Singularity + // load module script for Anaconda and Singularity beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } module = 'anaconda/5.3.1' module = 'singularity/3.5.3' From da3bd2693daf31531d0a57b6e046ea7b6e4a1d46 Mon Sep 17 00:00:00 2001 From: phue Date: Wed, 10 Mar 2021 11:26:01 +0100 Subject: [PATCH 15/44] cbe: add rapid qos and update config url --- conf/cbe.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 0a5763f..18f72dc 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -2,13 +2,13 @@ params { config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs' config_profile_contact = 'Patrick Hüther (@phue)' - config_profile_url = 'http://www.gmi.oeaw.ac.at/' + config_profile_url = 'https://clip.science' } process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } - clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } + clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } module = 'anaconda3/2019.10' } From e7f8189dfc95b07e34f8ea33ac51cc17d7a79d10 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 13:52:11 -0500 Subject: [PATCH 16/44] Create wcm.config --- conf/wcm.config | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 conf/wcm.config diff --git a/conf/wcm.config b/conf/wcm.config new file mode 100644 index 0000000..38cd3d1 --- /dev/null +++ b/conf/wcm.config @@ -0,0 +1,28 @@ +singularityDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow" + +params { + config_profile_description = 'Weill Cornell Medicine, Scientific Computing Unit Slurm cluster profile provided by nf-core/configs' + config_profile_contact = 'Ashley Stephen Doane, PhD (@DoaneAS)' + igenomes_base = '/athena/elementolab/scratch/reference/igenomes' +} + +singularity { + enabled = true + envWhitelist='SINGULARITY_BINDPATH' + cacheDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow" + autoMounts = true +} + +process { + executor = 'slurm' + queue = 'panda_physbio' + scratch = true + scratch = '/scratchLocal/`whoami`_${SLURM_JOBID}' +} + +params { + max_memory = 32.GB + max_cpus = 8 + max_time = 24.h +} + From d630bcfb6c12f92a54907de7b821a60a6f218a97 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 13:59:55 -0500 Subject: [PATCH 17/44] Create wcm.md --- docs/wcm.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/wcm.md diff --git a/docs/wcm.md b/docs/wcm.md new file mode 100644 index 0000000..74918ac --- /dev/null +++ b/docs/wcm.md @@ -0,0 +1,24 @@ +# nf-core/configs: Weill Cornell Medicine Configuration + +All nf-core pipelines have been successfully configured for use on the panda cluster at the WCM. + +To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Running the workflow on the Pasteur cluster + +Nextflow is not installed by default on the WCM cluster. + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +Nextflow manages each process as a separate job that is submitted to the cluster by using the `sbatch` command. +Nextflow shouldn't run directly on a login node but on a compute node or lab-specific interactive server when configured as a submit host. + +1. Run nextflow on a compute node or interactive server with submit host capability: + +```bash +# Run nextflow workflow +nextflow run \\ +nf-core/chipseq \\ +-resume \\ +-profile test,wcm +``` From 6343a7f87cbeaa24a81aa588998efefd0099ca57 Mon Sep 17 00:00:00 2001 From: Ashley S Doane Date: Thu, 11 Mar 2021 14:01:07 -0500 Subject: [PATCH 18/44] Update wcm.md --- docs/wcm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/wcm.md b/docs/wcm.md index 74918ac..0785829 100644 --- a/docs/wcm.md +++ b/docs/wcm.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the panda cluster at the WCM. -To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the WCM slurm cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. ## Running the workflow on the Pasteur cluster From cfe13fe6c1b72bbfeb27d56843c93f7714cc505d Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Mon, 15 Mar 2021 15:42:30 +0000 Subject: [PATCH 19/44] Fixed typo & added singularity tmpdir --- conf/eddie.config | 44 +++++++++++++++++++++++++------------------- nfcore_custom.config | 2 +- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 3968783..6145908 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -5,39 +5,45 @@ params { config_profile_url = 'https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing' } -conda { - createTimeout = '2h' -} - -singularity { - enabled = true +executor { + name = "sge" + queueSize = "100" } process { - executor = 'sge' - - // memory environment & options - clusterOptions = {"-l h_vmem=${task.memory.bytes/task.cpus}"} - penv = 'sharedmem' + clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus}" : null } + scratch = true + penv = { task.cpus > 1 ? "sharedmem" : null } // common SGE error statuses - errorStrategy = {task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish'} + errorStrategy = {task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish'} maxErrors = '-1' maxRetries = 3 - // load module script for Anaconda and Singularity - beforeScript = { '. /etc/profile.d/modules.sh; sleep 2; ' } - module = 'anaconda/5.3.1' - module = 'singularity/3.5.3' + beforeScript = + """ + '. /etc/profile.d/modules.sh + sleep 2 + module load 'anaconda/5.3.1' + module load 'roslin/singularity/3.5.3' + export SINGULARITY_TMPDIR="\$TMPDIR" + """ } params { saveReference = true - // iGenomes reference base - igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes/' - + igenomes_base = '/exports/igmm/eddie/NextGenResources/igenomes' max_memory = 384.GB max_cpus = 32 max_time = 240.h } + +env { + MALLOC_ARENA_MAX=1 +} + +singularity { + envWhitelist = "SINGULARITY_TMPDIR" + runOptions = '-p' +} \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 7abbb7e..b338f1c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_bsae}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 7e9f83ee8133993c4d14b1c1562644427e51655b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 15:54:19 +0100 Subject: [PATCH 20/44] Add IFB config --- .github/workflows/main.yml | 2 +- conf/ifb_core.config | 24 +++++++++++++++++++++++ docs/ifb_core.md | 40 ++++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 conf/ifb_core.config create mode 100644 docs/ifb_core.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3e5c930..cf00bea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/conf/ifb_core.config b/conf/ifb_core.config new file mode 100644 index 0000000..50a72f4 --- /dev/null +++ b/conf/ifb_core.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The IFB core cluster profile' + config_profile_contact = 'https://community.france-bioinformatique.fr' + config_profile_url = 'https://www.france-bioinformatique.fr/' +} + +singularity { + // need one image per execution + enabled = true + runOptions = '-B /shared' +} + +process { + executor = 'slurm' +} + +params { + igenomes_ignore = true + // Max resources requested by a normal node on genotoul. + max_memory = 240.GB + max_cpus = 28 + max_time = 96.h +} diff --git a/docs/ifb_core.md b/docs/ifb_core.md new file mode 100644 index 0000000..0434a09 --- /dev/null +++ b/docs/ifb_core.md @@ -0,0 +1,40 @@ +# nf-core/configs: IFB core Configuration + +All nf-core pipelines have been successfully configured for use on the IFB core at the insert institution here. + +To use, run the pipeline with `-profile ifb_core`. This will download and launch the [`ifb_core.config`](../conf/ifb_core.config) which has been pre-configured with a setup suitable for the IFB core cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## How to use on IFB core + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on IFB core. You can do this by issuing the commands below: + +```bash +# Login to a compute node +srun --pty bash + +## Load Nextflow and Singularity environment modules +module purge +module load nextflow/20.04.1 + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +nf-core/workflow \\ +-resume +-profile ifb_core \\ +--email my-email@example.org \\ +-c my-specific.config +... + + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... + +``` + +## Databanks + +A local copy of several genomes are available in `/shared/bank` directory. See +our [databank page](https://ifb-elixirfr.gitlab.io/cluster/doc/banks/) +to search for your favorite genome. + +>NB: You will need an account to use the HPC cluster on IFB core in order to run the pipeline. If in doubt contact IT or go to [account page](https://my.cluster.france-bioinformatique.fr/manager2/login). diff --git a/nfcore_custom.config b/nfcore_custom.config index 429e3c1..a4c9939 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,6 +24,7 @@ profiles { czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } + ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } From cc17045914a56ec05d06da080bc536d20588631a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 15:58:29 +0100 Subject: [PATCH 21/44] Add institution --- docs/ifb_core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ifb_core.md b/docs/ifb_core.md index 0434a09..3c1f546 100644 --- a/docs/ifb_core.md +++ b/docs/ifb_core.md @@ -1,6 +1,6 @@ # nf-core/configs: IFB core Configuration -All nf-core pipelines have been successfully configured for use on the IFB core at the insert institution here. +All nf-core pipelines have been successfully configured for use on the cluster of the IFB (Institut Francais de Bioinformatique). To use, run the pipeline with `-profile ifb_core`. This will download and launch the [`ifb_core.config`](../conf/ifb_core.config) which has been pre-configured with a setup suitable for the IFB core cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. From e3216ccf79aa60edcb8edd30fa8e2618171de90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 16 Mar 2021 16:00:56 +0100 Subject: [PATCH 22/44] typo --- docs/ifb_core.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ifb_core.md b/docs/ifb_core.md index 3c1f546..90c0eb1 100644 --- a/docs/ifb_core.md +++ b/docs/ifb_core.md @@ -6,7 +6,7 @@ To use, run the pipeline with `-profile ifb_core`. This will download and launch ## How to use on IFB core -Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on IFB core. You can do this by issuing the commands below: +Before running the pipeline you will need to load Nextflow using the environment module system on IFB core. You can do this by issuing the commands below: ```bash # Login to a compute node From d4cd07c646dabcbaa70ccdd3c02647f8c1b34e86 Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Wed, 17 Mar 2021 09:21:26 +0000 Subject: [PATCH 23/44] Typo in beforescript --- conf/eddie.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/eddie.config b/conf/eddie.config index 6145908..17ddbba 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -22,7 +22,7 @@ process { beforeScript = """ - '. /etc/profile.d/modules.sh + . /etc/profile.d/modules.sh sleep 2 module load 'anaconda/5.3.1' module load 'roslin/singularity/3.5.3' From 2ce1e2a9cc10ef5f1fc8389fd31e6f4fb7eab402 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 17 Mar 2021 12:32:47 +0000 Subject: [PATCH 24/44] Create rnaseq.config --- pipeline/rnaseq.config | 1 + 1 file changed, 1 insertion(+) create mode 100644 pipeline/rnaseq.config diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/pipeline/rnaseq.config @@ -0,0 +1 @@ + From 4ad6bb2bec13d3bf7d4d256c6e2f23e789e5c92d Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 17 Mar 2021 14:50:40 +0000 Subject: [PATCH 25/44] Singularity info --- docs/eddie.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index e6a2f87..fec6539 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -11,10 +11,10 @@ There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https: To use, run the pipeline with `-profile eddie` (one hyphen). This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). -The configuration file supports running nf-core pipelines with either a Conda environment or Docker containers running under Singularity. +The configuration file supports running nf-core pipelines with Docker containers running under Singularity. Support for Conda will follow. ```bash -nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie,singularity # ..rest of pipeline flags ``` Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. @@ -30,11 +30,19 @@ To load the most recent version: module load igmm/apps/nextflow ``` - This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. To set up Nextflow on a login node ... TODO +## Singularity set-up + +Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. + +``` +module load singularity +export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity/nf-core-rnaseq_v3.0" +``` + ## Using iGenomes references A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. From 059d353a1e1d14c4307c5427a65115c7d85b2d6e Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Thu, 18 Mar 2021 12:31:56 +0000 Subject: [PATCH 26/44] Removed Conda, added automount Singularity --- conf/eddie.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/eddie.config b/conf/eddie.config index 17ddbba..70e0dcc 100644 --- a/conf/eddie.config +++ b/conf/eddie.config @@ -23,8 +23,6 @@ process { beforeScript = """ . /etc/profile.d/modules.sh - sleep 2 - module load 'anaconda/5.3.1' module load 'roslin/singularity/3.5.3' export SINGULARITY_TMPDIR="\$TMPDIR" """ @@ -46,4 +44,7 @@ env { singularity { envWhitelist = "SINGULARITY_TMPDIR" runOptions = '-p' -} \ No newline at end of file + enabled = true + autoMounts = true +} + From af6a6b1060d5b59c7e7be33b832701ada7ce2c27 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 18 Mar 2021 12:48:48 +0000 Subject: [PATCH 27/44] Details on running on eddie Removed Conda support, Singularity by default. Added login and screen session on wild west node options. --- docs/eddie.md | 60 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index fec6539..faecad7 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -11,13 +11,14 @@ There is a Slack channel dedicated to eddie users on the MRC IGMM Slack: [https: To use, run the pipeline with `-profile eddie` (one hyphen). This will download and launch the [`eddie.config`](../conf/eddie.config) which has been pre-configured with a setup suitable for the [University of Edinburgh Eddie HPC](https://www.ed.ac.uk/information-services/research-support/research-computing/ecdf/high-performance-computing). -The configuration file supports running nf-core pipelines with Docker containers running under Singularity. Support for Conda will follow. +The configuration file supports running nf-core pipelines with Docker containers running under Singularity by default. Conda is not currently supported. ```bash -nextflow run nf-core/PIPELINE -profile eddie,singularity # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags ``` -Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. +If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. To list versions: @@ -30,19 +31,60 @@ To load the most recent version: module load igmm/apps/nextflow ``` -This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Conda or Singularity for software management. +This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. -To set up Nextflow on a login node ... TODO +## Singularity set-up -## Singularity set-up +Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. -Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. - -``` +```bash module load singularity export NXF_SINGULARITY_CACHEDIR="/exports/igmm/eddie/NextGenResources/nextflow/singularity/nf-core-rnaseq_v3.0" ``` +Singularity will create a directory `.singularity` in your `$HOME` directory on eddie. Space on `$HOME` is very limited, so it is a good idea to create a directory somewhere else with more room and link the locations. + +```bash +cd $HOME +mkdir /exports/eddie/path/to/my/area/.singularity +ln -s /exports/eddie/path/to/my/area/.singularity .singularity +``` + +## Running Nextflow + +### On a login node + +You can use a qlogin to run Nextflow, if you request more than the default 2GB of memory. Unfortunately you can't submit the initial Nextflow run process as a job as you can't qsub within a qsub. + +```bash +qlogin -l h_vmem=8G +``` + +If your eddie terminal disconnects your Nextflow job will stop. You can run Nextflow as a bash script on the command line using `nohup` to prevent this. + +``` +nohup ./nextflow_run.sh & +``` + +### On a wild west node + +Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). + +Start a new screen session. +```bash +screen -S +``` + +List existing screen sessions +```bash +screen -ls +``` + +Reconnect to an existing screen session +```bash +screen -r +``` + ## Using iGenomes references A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. From b48136a62e547aa2c8f337f76d4872ef4b80a224 Mon Sep 17 00:00:00 2001 From: ameynert Date: Thu, 18 Mar 2021 12:55:03 +0000 Subject: [PATCH 28/44] Added pipeline-specific config files --- docs/eddie.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index faecad7..5a8dfa5 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -14,7 +14,7 @@ This will download and launch the [`eddie.config`](../conf/eddie.config) which h The configuration file supports running nf-core pipelines with Docker containers running under Singularity by default. Conda is not currently supported. ```bash -nextflow run nf-core/PIPELINE -profile eddie # ..rest of pipeline flags +nextflow run nf-core/PIPELINE -profile eddie # ...rest of pipeline flags ``` Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. @@ -33,6 +33,14 @@ module load igmm/apps/nextflow This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. +### Pipeline-specific config files + +In addition to the generic nf-core configuration accessed via `-profile eddie`, there are pipeline-specific configuration files in the NextGenResources fileset to handle processes that are known to require specific custom configuration on the eddie system, in particular Java processes which require an extra memory overhead. For example: + +```bash +nextflow run nf-core/rnaseq -profile eddie -c /exports/igmm/eddie/NextGenResources/nextflow/conf/eddie.rnaseq.config # ...rest of pipeline flags +``` + ## Singularity set-up Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. From 93bde78faad71fbb97795b9fdbae1af79baa8f30 Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Sun, 21 Mar 2021 06:47:52 -0400 Subject: [PATCH 29/44] added JAX conf --- .github/workflows/main.yml | 2 +- README.md | 1 + nfcore_custom.config | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf00bea..41601d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/README.md b/README.md index 45721e8..5616780 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,7 @@ Currently documentation is available for the following systems: * [GOOGLE](docs/google.md) * [HEBBE](docs/hebbe.md) * [ICR_DAVROS](docs/icr_davros.md) +* [JAX](docs/jax.md) * [KRAKEN](docs/kraken.md) * [MPCDF](docs/mpcdf.md) * [MUNIN](docs/munin.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index a4c9939..3cafe07 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -46,6 +46,7 @@ profiles { uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } + jax { includeConfig "${params.custom_config_base}/conf/jax.config" } } // If user hostnames contain one of these substring and they are From 30c864e1121e5b5fcea5633eaa689498745670b2 Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Sun, 21 Mar 2021 06:49:11 -0400 Subject: [PATCH 30/44] added JAX conf --- conf/jax.config | 21 +++++++++++++++++++++ docs/jax.md | 8 ++++++++ 2 files changed, 29 insertions(+) create mode 100644 conf/jax.config create mode 100644 docs/jax.md diff --git a/conf/jax.config b/conf/jax.config new file mode 100644 index 0000000..a5437e9 --- /dev/null +++ b/conf/jax.config @@ -0,0 +1,21 @@ +params { + config_profile_description = 'The Jackson Laboratory Sumner HPC profile provided by nf-core/configs.' + config_profile_contact = 'Asaf Peer (@peera)' + config_profile_url = 'https://jacksonlaboratory.sharepoint.com/sites/ResearchIT/SitePages/Welcome-to-Sumner.aspx' + } + +executor.$slurm.queueSize = 250 +process { + executor = "slurm" + queue = "compute" + clusterOptions = {task.time < 72.h ? '-q batch' : '-q long'} + module = "slurm" + beforeScript = 'module load singularity' +} +singularity.enabled = true +singularity.autoMounts = true +params { + max_memory = 768.GB + max_cpus = 70 + max_time = 336.h + } diff --git a/docs/jax.md b/docs/jax.md new file mode 100644 index 0000000..d38cfbe --- /dev/null +++ b/docs/jax.md @@ -0,0 +1,8 @@ +# nf-core/configs: JAX Configuration + +All nf-core pipelines have been successfully configured for use on the JAX Sumner cluster at The Jackson Laboratory. + +To use, run the pipeline with `-profile jax`. This will download and launch the [`jax.config`](../conf/jax.config) which has been pre-configured with a setup suitable for JAX Sumner cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline and slurm will be used as well. + +>NB: You will need an account to use the HPC cluster JAX in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow should not be executed on the login nodes. If in doubt contact IT. From 4570718b12ef8b5b1e781a94bd3a270167d3d004 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Mar 2021 21:45:39 -0500 Subject: [PATCH 31/44] fix(ganymede): Add TMPDIR and SINGULARITY_CACHEDIR env variables --- conf/utd_ganymede.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index 409dc0a..f701a3c 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -5,6 +5,11 @@ params { config_profile_url = 'http://docs.oithpc.utdallas.edu/' } +env { + TMPDIR = '/home/$USER/scratch/tmp' + SINGULARITY_CACHEDIR = '/home/$USER/scratch/tmp' +} + singularity { enabled = true envWhitelist='SINGULARITY_BINDPATH' From 34ccce9531351fd8133c3dc3a77dab20398a5735 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Sun, 21 Mar 2021 21:46:25 -0500 Subject: [PATCH 32/44] fix(ganymede): Add process resources Trimgalore though it can be a bottom neck, doesn't require the 1 big memory node we have and can be run on the genomics queue. --- conf/utd_ganymede.config | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index f701a3c..c64e673 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -20,6 +20,28 @@ process { beforeScript = 'module load singularity/3.2.1' executor = 'slurm' queue = { task.memory >= 32.GB && task.cpu <= 12 ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' } + + withName:TRIMGALORE { + memory = 31.GB + } + + withLabel:process_low { + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } + } + + withLabel:process_medium { + cpus = { check_max( 16 * task.attempt, 'cpus' ) } + memory = { check_max( 31.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } + } + + withLabel:process_large { + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = { check_max( 16.h * task.attempt, 'time' ) } + } } params { From a6975cdd4ef05a552aaa6de492d88a525974a313 Mon Sep 17 00:00:00 2001 From: Edmund Miller Date: Mon, 22 Mar 2021 12:42:56 -0500 Subject: [PATCH 33/44] fix(ganymede): large => high Co-authored-by: drpatelh --- conf/utd_ganymede.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/utd_ganymede.config b/conf/utd_ganymede.config index c64e673..01bb10b 100644 --- a/conf/utd_ganymede.config +++ b/conf/utd_ganymede.config @@ -37,9 +37,9 @@ process { time = { check_max( 8.h * task.attempt, 'time' ) } } - withLabel:process_large { + withLabel:process_high { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 1.GB * task.attempt, 'memory' ) } + memory = { check_max( 120.GB * task.attempt, 'memory' ) } time = { check_max( 16.h * task.attempt, 'time' ) } } } From 791f3c95f34eb21d2ecd573b8611ba980f6bf0a3 Mon Sep 17 00:00:00 2001 From: ameynert Date: Tue, 23 Mar 2021 16:07:07 +0000 Subject: [PATCH 34/44] Pipeline specific configs moving to confs/pipeline --- docs/eddie.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 5a8dfa5..1a63568 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -33,14 +33,6 @@ module load igmm/apps/nextflow This config enables Nextflow to manage the pipeline jobs via the SGE job scheduler and using Singularity for software management. -### Pipeline-specific config files - -In addition to the generic nf-core configuration accessed via `-profile eddie`, there are pipeline-specific configuration files in the NextGenResources fileset to handle processes that are known to require specific custom configuration on the eddie system, in particular Java processes which require an extra memory overhead. For example: - -```bash -nextflow run nf-core/rnaseq -profile eddie -c /exports/igmm/eddie/NextGenResources/nextflow/conf/eddie.rnaseq.config # ...rest of pipeline flags -``` - ## Singularity set-up Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. From da8cf2f20705f4fca348c8a6185dada9430092b4 Mon Sep 17 00:00:00 2001 From: ameyner2 Date: Wed, 24 Mar 2021 10:46:26 +0000 Subject: [PATCH 35/44] Added pipeline-specific config files for rnaseq and sarek --- conf/pipeline/rnaseq/eddie.config | 15 +++++++++++++++ conf/pipeline/sarek/eddie.config | 27 +++++++++++++++++++++++++++ pipeline/rnaseq.config | 13 +++++++++++++ pipeline/sarek.config | 1 + 4 files changed, 56 insertions(+) create mode 100644 conf/pipeline/rnaseq/eddie.config create mode 100644 conf/pipeline/sarek/eddie.config create mode 100644 pipeline/rnaseq.config diff --git a/conf/pipeline/rnaseq/eddie.config b/conf/pipeline/rnaseq/eddie.config new file mode 100644 index 0000000..d8e76d0 --- /dev/null +++ b/conf/pipeline/rnaseq/eddie.config @@ -0,0 +1,15 @@ +process { + +withName : "PICARD_MARKDUPLICATES" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +withName : "QUALIMAP_RNASEQ" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +withName : "FASTQC" { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} +} + +} diff --git a/conf/pipeline/sarek/eddie.config b/conf/pipeline/sarek/eddie.config new file mode 100644 index 0000000..3e1b811 --- /dev/null +++ b/conf/pipeline/sarek/eddie.config @@ -0,0 +1,27 @@ +process { + + withName:MapReads { + cpus = 16 + } + withName:BuildDict { + cpus = 1 + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:BamQC { + cpus = 8 + memory = 128.GB + clusterOptions = {"-l h_vmem=${(task.memory + 8.GB).bytes/task.cpus}"} + } + withName:MarkDuplicates { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:BaseRecalibrator { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:ApplyBQSR { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } + withName:GatherBQSRReports { + clusterOptions = {"-l h_vmem=${(task.memory + 4.GB).bytes/task.cpus}"} + } +} diff --git a/pipeline/rnaseq.config b/pipeline/rnaseq.config new file mode 100644 index 0000000..1a27463 --- /dev/null +++ b/pipeline/rnaseq.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/rnaseq custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/rnaseq folder and imported + * under a profile name here. + */ + +profiles { + eddie { includeConfig "${params.custom_config_base}/conf/pipeline/rnaseq/eddie.config" } +} diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 4ef48da..d781478 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -12,4 +12,5 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" } + eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" } } \ No newline at end of file From 96fe6185a08cb12de4c510c22ba84974ba286f72 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:01:41 +0000 Subject: [PATCH 36/44] IGMM-specific docs --- docs/eddie.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 1a63568..363dcc5 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -35,7 +35,9 @@ This config enables Nextflow to manage the pipeline jobs via the SGE job schedul ## Singularity set-up -Load Singularity from the module system and set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. +Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. + +If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area which has limited space. It will take time to download all the Singularity containers, but you can use this again. ```bash module load singularity @@ -66,7 +68,7 @@ If your eddie terminal disconnects your Nextflow job will stop. You can run Next nohup ./nextflow_run.sh & ``` -### On a wild west node +### On a wild west node - IGMM only Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). @@ -87,7 +89,7 @@ screen -r ## Using iGenomes references -A local copy of the iGenomes resource has been made available on the Eddie HPC so you should be able to run the pipeline against any reference available in the `igenomes.config`. +A local copy of the iGenomes resource has been made available on the Eddie HPC for those with access to `/exports/igmm/eddie/NextGenResources` so you should be able to run the pipeline against any reference available in the `igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Adjusting maximum resources From 2a8ae2b2148c3782b601f1b613f3b1462fb8384f Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:12:41 +0000 Subject: [PATCH 37/44] Update eddie.md --- docs/eddie.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index 363dcc5..c51f7c1 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -37,7 +37,7 @@ This config enables Nextflow to manage the pipeline jobs via the SGE job schedul Load Singularity from the module system and, if you have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to the NextGenResources path for the pipeline and version you want to run. If this does not exist, please contact the [IGMM Data Manager](data.manager@igmm.ed.ac.uk) to have it added. You can add these lines to the file `$HOME/.bashrc`, or you can run these commands before you run an nf-core pipeline. -If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area which has limited space. It will take time to download all the Singularity containers, but you can use this again. +If you do not have access to `/exports/igmm/eddie/NextGenResources`, set the Singularity cache directory to somewhere sensible that is not in your `$HOME` area (which has limited space). It will take time to download all the Singularity containers, but you can use this again. ```bash module load singularity From aeb0ffcfdea232d81edc5c7a635f579235323247 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 11:26:09 +0000 Subject: [PATCH 38/44] Spacing fix Co-authored-by: Maxime Garcia --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index b338f1c..4fffc74 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 158402eddf6784d0e7d292d3bc8a76d26941d2fe Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 24 Mar 2021 12:26:49 +0100 Subject: [PATCH 39/44] Update nfcore_custom.config --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 4fffc74..9ea0197 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" } From 8e6223d1bdb341b5b00a990027a29c371ff5ed99 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 24 Mar 2021 12:36:50 +0100 Subject: [PATCH 40/44] Update nfcore_custom.config --- nfcore_custom.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 907e561..eb287b9 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -23,7 +23,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" } - eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } + eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" } icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From 2d90c919aad00e682a7bb78c50ee6f265e00064a Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:51:37 +0000 Subject: [PATCH 41/44] Added eddie profile to github workflow tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 41601d4..50dc704 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] + profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'eddie', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From a0d42f51536cbc21323c95afce0f19e9e7c1aa4a Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:55:31 +0000 Subject: [PATCH 42/44] Fixed code blocks for markdown linting --- docs/eddie.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/eddie.md b/docs/eddie.md index c51f7c1..283e21f 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -27,6 +27,7 @@ module avail igmm/apps/nextflow ``` To load the most recent version: + ```bash module load igmm/apps/nextflow ``` @@ -64,7 +65,7 @@ qlogin -l h_vmem=8G If your eddie terminal disconnects your Nextflow job will stop. You can run Nextflow as a bash script on the command line using `nohup` to prevent this. -``` +```bash nohup ./nextflow_run.sh & ``` @@ -73,16 +74,19 @@ nohup ./nextflow_run.sh & Wild west nodes on eddie can be accessed via ssh (node2c15, node2c16, node3g22). To run Nextflow on one of these nodes, do it within a [screen session](https://linuxize.com/post/how-to-use-linux-screen/). Start a new screen session. + ```bash screen -S ``` List existing screen sessions + ```bash screen -ls ``` Reconnect to an existing screen session + ```bash screen -r ``` From 3a8e54f153f249f7ed06dd9e6ea1e47eec137ca1 Mon Sep 17 00:00:00 2001 From: ameynert Date: Wed, 24 Mar 2021 13:58:10 +0000 Subject: [PATCH 43/44] Removed sneaky trailing whitespace. --- docs/eddie.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/eddie.md b/docs/eddie.md index 283e21f..cb78d71 100644 --- a/docs/eddie.md +++ b/docs/eddie.md @@ -17,8 +17,7 @@ The configuration file supports running nf-core pipelines with Docker containers nextflow run nf-core/PIPELINE -profile eddie # ...rest of pipeline flags ``` -Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. -If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. +Before running the pipeline you will need to install Nextflow or load it from the module system. Generally the most recent version will be the one you want. If you want to run a Nextflow pipeline that is based on [DSL2](https://www.nextflow.io/docs/latest/dsl2.html), you will need a version that ends with '-edge'. To list versions: From 922ab87eb0f69cf994220a4f15f008f9784d16fa Mon Sep 17 00:00:00 2001 From: Asaf Peer Date: Wed, 24 Mar 2021 11:25:47 -0400 Subject: [PATCH 44/44] added jax singularity cache dir --- conf/jax.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/jax.config b/conf/jax.config index a5437e9..7cf790e 100644 --- a/conf/jax.config +++ b/conf/jax.config @@ -2,6 +2,7 @@ params { config_profile_description = 'The Jackson Laboratory Sumner HPC profile provided by nf-core/configs.' config_profile_contact = 'Asaf Peer (@peera)' config_profile_url = 'https://jacksonlaboratory.sharepoint.com/sites/ResearchIT/SitePages/Welcome-to-Sumner.aspx' + singularity_cache_dir = '/fastscratch/singularity_cache_nfcore' } executor.$slurm.queueSize = 250 @@ -12,8 +13,11 @@ process { module = "slurm" beforeScript = 'module load singularity' } -singularity.enabled = true -singularity.autoMounts = true +singularity{ + enabled = true + autoMounts = true + cacheDir = params.singularity_cache_dir +} params { max_memory = 768.GB max_cpus = 70