From 55971bb94de2194f3a3eea7c5965d5dde1b9fef5 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 16 Sep 2022 17:29:08 +0100 Subject: [PATCH 01/13] Modularised error handling strategy to per pipeline basis --- conf/pipeline/atacseq/sbc_sharc.config | 9 +++++++++ conf/pipeline/chipseq/sbc_sharc.config | 9 +++++++++ conf/pipeline/rnaseq/sbc_sharc.config | 8 ++++++++ conf/pipeline/sarek/sbc_sharc.config | 7 ++++++- conf/sbc_sharc.config | 22 +++++++++------------- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index e50695c..0660591 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -15,6 +15,14 @@ process { + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels + withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } @@ -72,3 +80,4 @@ def check_max(obj, type) { } } } + diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 60912f3..6fef3ba 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -14,6 +14,14 @@ // process-specific resource requirements - reduced specification from those in chipseq/conf/base.config process { + + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } @@ -72,3 +80,4 @@ def check_max(obj, type) { } } } + diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 52bf0ff..8cefa40 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -14,6 +14,14 @@ // process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config process { + + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } + maxRetries = 2 + + + // process labels withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 204d73b..434b8a2 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -14,8 +14,13 @@ // process-specific resource requirements - reduced specification from those in sarek/conf/base.config process { - + // error and retry handling + + errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } + maxRetries = 2 + + // process labels withLabel:process_low { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 20b8661..de1ab0c 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -28,16 +28,6 @@ params { } -// container engine - -singularity { - - enabled = true - autoMounts = true - -} - - // hpc configuration specific to ShARC process { @@ -49,9 +39,15 @@ process { queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } clusterOptions = { "-l rmem=${task.memory.toGiga()}G" } - // error and retry handling +} - errorStrategy = { task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish' } - maxRetries = 2 + +// container engine + +singularity { + + enabled = true + autoMounts = true } + From adc88cfa1f3edec886261c5f721cb97581a019ac Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Mon, 19 Sep 2022 16:30:30 +0100 Subject: [PATCH 02/13] Added config for limiting queue size and job submit rate --- conf/sbc_sharc.config | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index de1ab0c..bf00d33 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -11,16 +11,21 @@ */ +// nf-core specific parameters displayed in header summary of each run + params { - - // nf-core specific parameters displayed in header summary of each run config_profile_description = 'Sheffield Bioinformatics Core - ShARC' config_profile_contact = 'Lewis Quayle (l.quayle@sheffield.ac.uk)' config_profile_url = 'https://docs.hpc.shef.ac.uk/en/latest/sharc/index.html' - - // hpc resource limits +} + + +// hpc resource limits + +params { + max_cpus = 16 max_memory = 64.GB max_time = 96.h @@ -42,6 +47,16 @@ process { } +// optional executor settings + +executor { + + queueSize = 10 + submitRateLimit = '1 sec' + +} + + // container engine singularity { @@ -50,4 +65,3 @@ singularity { autoMounts = true } - From 1471257aaf2f6253b97bc3133c18f990e96dfb35 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 20 Sep 2022 19:51:31 +0100 Subject: [PATCH 03/13] Modified config for job submit rate --- conf/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index bf00d33..a8a096e 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -52,7 +52,7 @@ process { executor { queueSize = 10 - submitRateLimit = '1 sec' + submitRateLimit = '12/1min' } From 7e34beb3c9d75f580888b2b025aa684206df27ca Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 11:08:39 +0100 Subject: [PATCH 04/13] Updated resource requests for compatibility with general worker nodes --- conf/pipeline/atacseq/sbc_sharc.config | 2 +- conf/pipeline/chipseq/sbc_sharc.config | 2 +- conf/pipeline/rnaseq/sbc_sharc.config | 6 +----- conf/pipeline/sarek/sbc_sharc.config | 10 +++++----- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 0660591..97cd758 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 6fef3ba..a2f6431 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8cefa40..8f905f7 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -45,10 +45,6 @@ process { time = { check_max( 12.h * task.attempt, 'time' ) } } - withLabel:process_high_memory { - memory = { check_max( 60.GB * task.attempt, 'memory' ) } - } - } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 434b8a2..24312a7 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -37,7 +37,7 @@ process { withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +46,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 60.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } @@ -54,7 +54,7 @@ process { withName:'BWAMEM1_MEM|BWAMEM2_MEM' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -71,11 +71,11 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From ab1ad832dfed33016503f7011a1ae4d732a68922 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 13:18:50 +0100 Subject: [PATCH 05/13] Updated resource requests for compatibility with high memory nodes --- conf/pipeline/atacseq/sbc_sharc.config | 12 ++++++------ conf/pipeline/chipseq/sbc_sharc.config | 12 ++++++------ conf/pipeline/rnaseq/sbc_sharc.config | 16 ++++++++++------ conf/pipeline/sarek/sbc_sharc.config | 24 ++++++++++++------------ conf/sbc_sharc.config | 4 ++-- 5 files changed, 36 insertions(+), 32 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 97cd758..60cbc49 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index a2f6431..10de65b 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8f905f7..fe50bc6 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -45,6 +45,10 @@ process { time = { check_max( 12.h * task.attempt, 'time' ) } } + withLabel:process_high_memory { + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + } + } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 24312a7..75e2be0 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -24,20 +24,20 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + cpus = { check_max( 5 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 10 * task.attempt, 'cpus' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,20 +46,20 @@ process { } withLabel:process_high_memory { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } // process name withName:'BWAMEM1_MEM|BWAMEM2_MEM' { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FASTP' { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } + cpus = { check_max( 15 * task.attempt, 'cpus' ) } } withName:'FASTQC|FASTP|MOSDEPTH|SAMTOOLS_CONVERT' { @@ -71,11 +71,11 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index a8a096e..21dd18b 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -27,7 +27,7 @@ params { params { max_cpus = 16 - max_memory = 64.GB + max_memory = 256.GB max_time = 96.h } @@ -52,7 +52,7 @@ process { executor { queueSize = 10 - submitRateLimit = '12/1min' + submitRateLimit = '1 sec' } From c42a5a9a94df4845630f68633c4401c8ce83269b Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 23 Sep 2022 17:01:58 +0100 Subject: [PATCH 06/13] Updated resources requested by pipelines --- conf/pipeline/atacseq/sbc_sharc.config | 10 +++++----- conf/pipeline/chipseq/sbc_sharc.config | 10 +++++----- conf/pipeline/rnaseq/sbc_sharc.config | 10 +++++----- conf/pipeline/sarek/sbc_sharc.config | 10 +++++----- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 60cbc49..0660591 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 10de65b..6fef3ba 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index fe50bc6..8033609 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } + memory = { check_max( 4.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } + cpus = { check_max( 4 * task.attempt, 'cpus' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 75e2be0..44dc57d 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -24,19 +24,19 @@ process { // process labels withLabel:process_low { - cpus = { check_max( 5 * task.attempt, 'cpus' ) } + cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { - cpus = { check_max( 10 * task.attempt, 'cpus' ) } + cpus = { check_max( 6 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -53,13 +53,13 @@ process { // process name withName:'BWAMEM1_MEM|BWAMEM2_MEM' { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FASTP' { - cpus = { check_max( 15 * task.attempt, 'cpus' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } } withName:'FASTQC|FASTP|MOSDEPTH|SAMTOOLS_CONVERT' { From 8103718389494dd91050ee160c1b7b4e2bd67749 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Fri, 30 Sep 2022 16:54:48 +0100 Subject: [PATCH 07/13] Reduced maxRetries upon error in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 44dc57d..95b3ee3 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -18,7 +18,7 @@ process { // error and retry handling errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } - maxRetries = 2 + maxRetries = 1 // process labels From dbb3b3b44b3205fb4c1c034da4d73159b346cf37 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 07:43:46 +0100 Subject: [PATCH 08/13] Reinstated maxRetries and increased memory for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 95b3ee3..4ddf2ba 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -18,7 +18,7 @@ process { // error and retry handling errorStrategy = { task.exitStatus in [143,137,104,134,139,140,247] ? 'retry' : 'finish' } - maxRetries = 1 + maxRetries = 2 // process labels @@ -75,7 +75,7 @@ process { } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 20.GB * task.attempt, 'memory' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From 10c14902650bcb7f448b660938581d6bc8ee5500 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 23:15:13 +0100 Subject: [PATCH 09/13] Increased memory and time for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 4ddf2ba..570284d 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -76,6 +76,7 @@ process { withName:'GATK4_MARKDUPLICATES' { memory = { check_max( 20.GB * task.attempt, 'memory' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { From 9b451b53b7a539c98cf91d25642792318146b0c2 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sat, 1 Oct 2022 23:31:51 +0100 Subject: [PATCH 10/13] Increased cpus and memory for GATK4_MARKDUPLICATES in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 570284d..170b8bd 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -26,13 +26,13 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 6 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 6.h * task.attempt, 'time' ) } + time = { check_max( 8.h * task.attempt, 'time' ) } } withLabel:process_high { @@ -75,7 +75,8 @@ process { } withName:'GATK4_MARKDUPLICATES' { - memory = { check_max( 20.GB * task.attempt, 'memory' ) } + cpus = { check_max( 8 * task.attempt, 'cpus' ) } + memory = { check_max( 30.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From e7ac3ecd1934b8554ec28c02e00998e0e4869d22 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Sun, 2 Oct 2022 00:24:39 +0100 Subject: [PATCH 11/13] Modified resource requests in sbc_sharc.config for sarek --- conf/pipeline/sarek/sbc_sharc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 170b8bd..d970574 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -76,7 +76,7 @@ process { withName:'GATK4_MARKDUPLICATES' { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 30.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } From 3f2ffa314d2e956eaf5c077aa3cc35d0fc25ad45 Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 4 Oct 2022 00:25:16 +0100 Subject: [PATCH 12/13] Revised base and sarek configs to provide memory on per core basis --- conf/pipeline/sarek/sbc_sharc.config | 20 ++++++++++---------- conf/sbc_sharc.config | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index d970574..4ef0534 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -25,19 +25,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { check_max( 6.h * task.attempt, 'time' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + memory = { check_max( 72.GB * task.attempt, 'memory' ) } + time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 192.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +46,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 240.GB * task.attempt, 'memory' ) } } @@ -54,7 +54,7 @@ process { withName:'BWAMEM1_MEM|BWAMEM2_MEM' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 192.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -71,12 +71,12 @@ process { } withName:'GATK4_APPLYBQSR|GATK4_APPLYBQSR_SPARK|GATK4_BASERECALIBRATOR|GATK4_GATHERBQSRREPORTS' { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 72.GB * task.attempt, 'memory' ) } } withName:'GATK4_MARKDUPLICATES' { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + cpus = { check_max( 12 * task.attempt, 'cpus' ) } + memory = { check_max( 240.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 21dd18b..6583da7 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -42,7 +42,7 @@ process { executor = 'sge' penv = 'smp' queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } - clusterOptions = { "-l rmem=${task.memory.toGiga()}G" } + clusterOptions = { "-l rmem=${task.memory.toGiga()/task.cpus}G" } } From 584052cfff0713b3659cbd85b8826603c378128b Mon Sep 17 00:00:00 2001 From: lquayle88 Date: Tue, 4 Oct 2022 09:12:29 +0100 Subject: [PATCH 13/13] Revised base and all pipeline configs to provide memory on per core basis --- conf/pipeline/atacseq/sbc_sharc.config | 20 +++++--------------- conf/pipeline/chipseq/sbc_sharc.config | 20 +++++--------------- conf/pipeline/rnaseq/sbc_sharc.config | 22 ++++++---------------- conf/pipeline/sarek/sbc_sharc.config | 16 +++------------- conf/sbc_sharc.config | 16 ++++------------ 5 files changed, 23 insertions(+), 71 deletions(-) diff --git a/conf/pipeline/atacseq/sbc_sharc.config b/conf/pipeline/atacseq/sbc_sharc.config index 0660591..7cd0e4a 100644 --- a/conf/pipeline/atacseq/sbc_sharc.config +++ b/conf/pipeline/atacseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/atacseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/atacseq // process-specific resource requirements - reduced specification from those in atacseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/chipseq/sbc_sharc.config b/conf/pipeline/chipseq/sbc_sharc.config index 6fef3ba..0e66333 100644 --- a/conf/pipeline/chipseq/sbc_sharc.config +++ b/conf/pipeline/chipseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/chipseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/chipseq // process-specific resource requirements - reduced specification from those in chipseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } diff --git a/conf/pipeline/rnaseq/sbc_sharc.config b/conf/pipeline/rnaseq/sbc_sharc.config index 8033609..d9d9878 100644 --- a/conf/pipeline/rnaseq/sbc_sharc.config +++ b/conf/pipeline/rnaseq/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/rnaseq - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/rnaseq // process-specific resource requirements - reduced specification from those in rnaseq/conf/base.config @@ -25,19 +15,19 @@ process { withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 4.GB * task.attempt, 'memory' ) } + memory = { check_max( 12.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } } withLabel:process_medium { cpus = { check_max( 4 * task.attempt, 'cpus' ) } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } time = { check_max( 6.h * task.attempt, 'time' ) } } withLabel:process_high { cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 128.GB * task.attempt, 'memory' ) } time = { check_max( 8.h * task.attempt, 'time' ) } } @@ -46,7 +36,7 @@ process { } withLabel:process_high_memory { - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 160.GB * task.attempt, 'memory' ) } } } diff --git a/conf/pipeline/sarek/sbc_sharc.config b/conf/pipeline/sarek/sbc_sharc.config index 4ef0534..3a7f5b9 100644 --- a/conf/pipeline/sarek/sbc_sharc.config +++ b/conf/pipeline/sarek/sbc_sharc.config @@ -1,15 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Custom Pipeline Resource Config for nf-core/sarek - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Custom Pipeline Resource Config for nf-core/sarek // process-specific resource requirements - reduced specification from those in sarek/conf/base.config @@ -77,7 +67,7 @@ process { withName:'GATK4_MARKDUPLICATES' { cpus = { check_max( 12 * task.attempt, 'cpus' ) } memory = { check_max( 240.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + time = { check_max( 12.h * task.attempt, 'time' ) } } withName:'FREEBAYES|SAMTOOLS_STATS|SAMTOOLS_INDEX|UNZIP' { diff --git a/conf/sbc_sharc.config b/conf/sbc_sharc.config index 6583da7..91cf4e3 100644 --- a/conf/sbc_sharc.config +++ b/conf/sbc_sharc.config @@ -1,14 +1,5 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Sheffield Bioinformatics Core Configuration Profile - ShARC - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Base Institutional Configuration - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ +// Sheffield Bioinformatics Core Configuration Profile - ShARC +// Base Institutional Configuration // nf-core specific parameters displayed in header summary of each run @@ -42,7 +33,7 @@ process { executor = 'sge' penv = 'smp' queue = { task.time <= 6.h ? 'shortint.q' : 'all.q' } - clusterOptions = { "-l rmem=${task.memory.toGiga()/task.cpus}G" } + clusterOptions = { "-l rmem=${ (task.memory.toGiga() / task.cpus) }G" } } @@ -65,3 +56,4 @@ singularity { autoMounts = true } +