From df0429ce9d72ae02c0a319d4e14325707cc7b517 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 3 Jun 2022 14:20:39 +0200 Subject: [PATCH 1/7] Replace oracle specific _JAVA_OPTIONS to portable JAVA_TOOLS_OPTIONS --- conf/pipeline/eager/eva.config | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index f35f1ed..f633bae 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -9,7 +9,7 @@ params { // Specific nf-core/eager process configuration process { - beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1"' + beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' maxRetries = 2 @@ -69,17 +69,17 @@ process { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName: fastqc_after_clipping { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } - errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: adapter_removal { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName: bwa { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga())}G,h=!(bionode01|bionode02|bionode03|bionode04|bionode05|bionode06)" } } @@ -188,19 +188,19 @@ process { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName:get_software_versions { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } - beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -Xmx512m"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' + beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1 -Xmx512m"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega())}M" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName:eigenstrat_snp_coverage { beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - } - + } + withName:kraken_merge { beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' } @@ -226,7 +226,7 @@ profiles { process { - beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1"' + beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' maxRetries = 2 @@ -279,7 +279,7 @@ profiles { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName: fastqc_after_clipping { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } @@ -404,7 +404,7 @@ profiles { process { - beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1"' + beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' maxRetries = 2 @@ -457,7 +457,7 @@ profiles { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - + withName: fastqc_after_clipping { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } From c40ebb9acfa5f46b98654d3adeae5c8057d01a96 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 3 Jun 2022 14:21:41 +0200 Subject: [PATCH 2/7] Add overhead for get_software_versions --- conf/pipeline/eager/eva.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index f633bae..92ce500 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -192,8 +192,8 @@ process { withName:get_software_versions { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } - beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1 -Xmx512m"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega())}M" } + beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' + clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 2)}M" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } From 9ef52af084fd8b9fa21d047022bf993ffe55b55f Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 3 Jun 2022 14:24:27 +0200 Subject: [PATCH 3/7] Increase overhead --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 92ce500..55b254c 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -193,7 +193,7 @@ process { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 2)}M" } + clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 4)}M" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } From 05cc8a08f133c66ff724b4032a3810d5549fe59b Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Jun 2022 21:08:19 +0200 Subject: [PATCH 4/7] Update eva.config --- conf/pipeline/eager/eva.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 55b254c..fa045b1 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -193,7 +193,7 @@ process { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 4)}M" } + clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 8)}M" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } @@ -207,6 +207,7 @@ process { withName:multiqc { beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1;' + clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" } } } From 46fba9c5970ca4ff7a235dfe3c9943ec58bd09d6 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Jun 2022 21:10:48 +0200 Subject: [PATCH 5/7] Update eva.config --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index fa045b1..a50c969 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -207,7 +207,7 @@ process { withName:multiqc { beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1;' - clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga()}G" } + clusterOptions = { "-S /bin/bash -V -j y -o output.log -l h_vmem=${task.memory.toGiga() * 2}G" } } } From 58dd9d49c7532ca114ce7dba2b8d6182806a4419 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 15 Jun 2022 15:21:20 +0200 Subject: [PATCH 6/7] Move beforeScript exports to env scope --- conf/pipeline/eager/eva.config | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 55b254c..a82257a 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -6,11 +6,15 @@ params { config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs' } +env { + JAVA_TOOL_OPTIONS = "-XX:ParallelGCThreads=1" + OPENBLAS_NUM_THREADS = 1 + OMP_NUM_THREADS = 1 +} + // Specific nf-core/eager process configuration process { - beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' - maxRetries = 2 // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion @@ -192,22 +196,10 @@ process { withName:get_software_versions { cache = false clusterOptions = { "-S /bin/bash -V -l h=!(bionode06)" } - beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"; export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toMega() * 4)}M" } errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } - withName:eigenstrat_snp_coverage { - beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - } - - withName:kraken_merge { - beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1' - } - - withName:multiqc { - beforeScript = 'export OPENBLAS_NUM_THREADS=1; export OMP_NUM_THREADS=1;' - } } profiles { @@ -226,8 +218,6 @@ profiles { process { - beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' - maxRetries = 2 // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion @@ -404,8 +394,6 @@ profiles { process { - beforeScript = 'export JAVA_TOOL_OPTIONS="-XX:ParallelGCThreads=1"' - maxRetries = 2 // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion From acabcbd9e8779305bf7470e22c935900085615ce Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 15 Jun 2022 15:27:40 +0200 Subject: [PATCH 7/7] Use _JAVA_OPTIONS instead --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 9f1853f..8376719 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -7,7 +7,7 @@ params { } env { - JAVA_TOOL_OPTIONS = "-XX:ParallelGCThreads=1" + _JAVA_OPTIONS = "-XX:ParallelGCThreads=1" OPENBLAS_NUM_THREADS = 1 OMP_NUM_THREADS = 1 }