From c5070f3eba267bbf58862c489c6748045e837d31 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Wed, 7 Apr 2021 17:07:49 +0200 Subject: [PATCH 1/9] Use node queue when more memory than thin nodes have --- conf/uppmax.config | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index bc3d8c4..da25dfc 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -9,25 +9,48 @@ singularity { enabled = true } +def hostname = "hostname".execute().text.trim() + process { + // closure to create a suitable clusterOptions + def clusterOptionsCreator = { m -> + String base = "-A $params.project ${params.clusterOptions ?: ''}" + // Do not use -p node on irma or if a thin node/core is enough + if (m < 125.GB || hostname ==~ "i.*") { + return base + } + if (m < 250.GB) { + return base + " -p node -C mem256GB " + } + // Remaining cases use the largest available node (1 Tbyte for rackham, 512 Gbyte for others) + if (hostname ==~ "r.*") { + return base + " -p node -C mem1TB " + } + return base + " -p node -C mem512GB " + } + + executor = 'slurm' - clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } + clusterOptions = { clusterOptionsCreator(task.memory) } } params { save_reference = true - max_memory = 125.GB + max_memory = 970.GB max_cpus = 16 max_time = 240.h // illumina iGenomes reference file paths on UPPMAX igenomes_base = '/sw/data/uppnex/igenomes/' } -def hostname = "hostname".execute().text.trim() +if (hostname ==~ "b.*") { + params.max_memory = 500.GB +} -if (hostname ==~ "b.*" || hostname ==~ "s.*") { - params.max_memory = 109.GB +if (hostname ==~ "s.*") { + params.max_memory = 500.GB + params.max_time = 700.h } if (hostname ==~ "i.*") { From 4d64db85eb6a4611605451ebaf0405671255e9a0 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Wed, 7 Apr 2021 18:42:33 +0200 Subject: [PATCH 2/9] Determine cluster through slurm --- conf/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index da25dfc..80f8991 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -9,7 +9,7 @@ singularity { enabled = true } -def hostname = "hostname".execute().text.trim() +def hostname = "sinfo --local -N -h | head -1 | cut -f1 -d' ' ".execute().text.trim() process { // closure to create a suitable clusterOptions From fa1b4cb4120e33b029eaaffc33ad6e2c1e6fbcf9 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 8 Apr 2021 09:45:29 +0200 Subject: [PATCH 3/9] Move cluster options creating closure out of process scope --- conf/uppmax.config | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index 80f8991..b521934 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -11,25 +11,26 @@ singularity { def hostname = "sinfo --local -N -h | head -1 | cut -f1 -d' ' ".execute().text.trim() +// closure to create a suitable clusterOptions +def clusterOptionsCreator = { m -> + String base = "-A $params.project ${params.clusterOptions ?: ''}" + // Do not use -p node on irma or if a thin node/core is enough + if (m < 125.GB || hostname ==~ "i.*") { + return base + } + + if (m < 250.GB) { + return base + " -p node -C mem256GB " + } + + // Remaining cases use the largest available node (1 Tbyte for rackham, 512 Gbyte for others) + if (hostname ==~ "r.*") { + return base + " -p node -C mem1TB " + } + return base + " -p node -C mem512GB " +} + process { - // closure to create a suitable clusterOptions - def clusterOptionsCreator = { m -> - String base = "-A $params.project ${params.clusterOptions ?: ''}" - // Do not use -p node on irma or if a thin node/core is enough - if (m < 125.GB || hostname ==~ "i.*") { - return base - } - if (m < 250.GB) { - return base + " -p node -C mem256GB " - } - // Remaining cases use the largest available node (1 Tbyte for rackham, 512 Gbyte for others) - if (hostname ==~ "r.*") { - return base + " -p node -C mem1TB " - } - return base + " -p node -C mem512GB " - } - - executor = 'slurm' clusterOptions = { clusterOptionsCreator(task.memory) } } From be6ea0060f247092556ecff109164c4d02611166 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Thu, 8 Apr 2021 09:49:42 +0200 Subject: [PATCH 4/9] Wrap sinfo call in a try/catch and default to rackham on failure --- conf/uppmax.config | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index b521934..4509117 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -9,7 +9,12 @@ singularity { enabled = true } -def hostname = "sinfo --local -N -h | head -1 | cut -f1 -d' ' ".execute().text.trim() +def hostname = "r1" +try { + hostname = "sinfo --local -N -h | head -1 | cut -f1 -d' ' ".execute().text.trim() +} catch (java.io.IOException e) { + System.err.println("WARNING: Could not run sinfo to determine current cluster, defaulting to rackham") +} // closure to create a suitable clusterOptions def clusterOptionsCreator = { m -> From 57c26c13d9b7bf9cadc483d261f49f2a474a54c2 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 12 Apr 2021 19:21:58 +0200 Subject: [PATCH 5/9] Redo cases for simplification --- conf/uppmax.config | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index 4509117..9223cc6 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -43,19 +43,14 @@ process { params { save_reference = true - max_memory = 970.GB + max_memory = 500.GB max_cpus = 16 max_time = 240.h // illumina iGenomes reference file paths on UPPMAX igenomes_base = '/sw/data/uppnex/igenomes/' } -if (hostname ==~ "b.*") { - params.max_memory = 500.GB -} - if (hostname ==~ "s.*") { - params.max_memory = 500.GB params.max_time = 700.h } @@ -65,6 +60,7 @@ if (hostname ==~ "i.*") { if (hostname ==~ "r.*") { params.max_cpus = 20 + params.max_memory = 970.GB } profiles { From 7be4279718aab354830c495c8dd6d93766524d15 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 12 Apr 2021 19:31:41 +0200 Subject: [PATCH 6/9] Handle different (unexpected) sinfo returns in different cases --- conf/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index 9223cc6..c8b954d 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -11,7 +11,7 @@ singularity { def hostname = "r1" try { - hostname = "sinfo --local -N -h | head -1 | cut -f1 -d' ' ".execute().text.trim() + hostname = "sinfo --local -N -h | grep -F -v CLUSTER: | head -1 | cut -f1 -d' ' ".execute().text.trim() } catch (java.io.IOException e) { System.err.println("WARNING: Could not run sinfo to determine current cluster, defaulting to rackham") } From 0987cbe1beaf948b187703c0953cb625ee01f65e Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 12 Apr 2021 19:32:21 +0200 Subject: [PATCH 7/9] Better test for snowy --- conf/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index c8b954d..93fa32c 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -50,7 +50,7 @@ params { igenomes_base = '/sw/data/uppnex/igenomes/' } -if (hostname ==~ "s.*") { +if (hostname ==~ "s[0-9][0-9]*") { params.max_time = 700.h } From ff20f121f952a731b93f5719bc2006956117b9c1 Mon Sep 17 00:00:00 2001 From: Pontus Freyhult Date: Mon, 12 Apr 2021 19:34:28 +0200 Subject: [PATCH 8/9] Support use of very fat node in snowy --- conf/uppmax.config | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index 93fa32c..f10cdd8 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -28,10 +28,18 @@ def clusterOptionsCreator = { m -> return base + " -p node -C mem256GB " } - // Remaining cases use the largest available node (1 Tbyte for rackham, 512 Gbyte for others) + + // Use mem1TB for remaining cases on rackham (no 512 Gbyte nodes) if (hostname ==~ "r.*") { return base + " -p node -C mem1TB " } + + if (m > 500.GB) { + // Special case for snowy very fat node (only remaining case that's above 500 GB) + return base + " -p veryfat " + } + + // Should only be cases for mem512GB left (snowy and bianca) return base + " -p node -C mem512GB " } @@ -52,6 +60,7 @@ params { if (hostname ==~ "s[0-9][0-9]*") { params.max_time = 700.h + params.max_memory = 3880.GB } if (hostname ==~ "i.*") { From 747a89b4c5ca9c92ab4c54f8b049c1dc9883a1ea Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 8 Oct 2021 13:40:59 +0200 Subject: [PATCH 9/9] Update conf/uppmax.config --- conf/uppmax.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/uppmax.config b/conf/uppmax.config index f10cdd8..8706ede 100644 --- a/conf/uppmax.config +++ b/conf/uppmax.config @@ -55,7 +55,7 @@ params { max_cpus = 16 max_time = 240.h // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/sw/data/uppnex/igenomes/' + igenomes_base = '/sw/data/igenomes/' } if (hostname ==~ "s[0-9][0-9]*") {