From 8b9744723caf27536d42814ce4a150d31faba8f3 Mon Sep 17 00:00:00 2001 From: Emil Date: Wed, 4 May 2022 14:13:24 +0200 Subject: [PATCH 1/8] Fixes hasta java memory issues --- conf/hasta.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conf/hasta.config b/conf/hasta.config index c5dd5e4..7db9b4c 100644 --- a/conf/hasta.config +++ b/conf/hasta.config @@ -21,6 +21,14 @@ params { process { executor = 'slurm' clusterOptions = { "-A $params.priority ${params.clusterOptions ?: ''}" } + + withName: QUALIMAP_BAMQC { + memory = 64.GB + ext.args = "--java-mem-size=60G" + } + withName: 'PICARD_COLLECTHSMETRICS|PICARD_COLLECTMULTIPLEMETRICS|COLLECTREADCOUNTS|DENOISEREADCOUNTS' { + clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus}+2" : null } + } } profiles { From e938f3606f012c4bf583a26544236205fae83ed0 Mon Sep 17 00:00:00 2001 From: Emil Date: Thu, 5 May 2022 14:09:45 +0200 Subject: [PATCH 2/8] Removes the new SGE clusterOptions for now --- conf/hasta.config | 3 --- 1 file changed, 3 deletions(-) diff --git a/conf/hasta.config b/conf/hasta.config index 7db9b4c..67d8652 100644 --- a/conf/hasta.config +++ b/conf/hasta.config @@ -26,9 +26,6 @@ process { memory = 64.GB ext.args = "--java-mem-size=60G" } - withName: 'PICARD_COLLECTHSMETRICS|PICARD_COLLECTMULTIPLEMETRICS|COLLECTREADCOUNTS|DENOISEREADCOUNTS' { - clusterOptions = { task.memory ? "-l h_vmem=${task.memory.bytes/task.cpus}+2" : null } - } } profiles { From 19497a4183123c8694093889ace39fd162ca7b44 Mon Sep 17 00:00:00 2001 From: Emil Date: Mon, 30 May 2022 11:06:48 +0200 Subject: [PATCH 3/8] Makes qualimap fix work for most memory amounts --- conf/hasta.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/hasta.config b/conf/hasta.config index 67d8652..b83f906 100644 --- a/conf/hasta.config +++ b/conf/hasta.config @@ -23,8 +23,7 @@ process { clusterOptions = { "-A $params.priority ${params.clusterOptions ?: ''}" } withName: QUALIMAP_BAMQC { - memory = 64.GB - ext.args = "--java-mem-size=60G" + ext.args = { "--java-mem-size=${task.memory.giga / 1.15 as long}G" } } } From 29fa08c43439bd277d3ac2fc72826bf0cbc8641a Mon Sep 17 00:00:00 2001 From: Emil Date: Mon, 30 May 2022 11:07:24 +0200 Subject: [PATCH 4/8] Adds _JAVA_OPTIONS to singularity whitelist --- conf/hasta.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/hasta.config b/conf/hasta.config index b83f906..862676a 100644 --- a/conf/hasta.config +++ b/conf/hasta.config @@ -10,6 +10,7 @@ params { singularity { enabled = true + envWhitelist = ['_JAVA_OPTIONS'] } params { From 8453f6b41a31146d512694175490b9e478a44a0d Mon Sep 17 00:00:00 2001 From: Emil Date: Mon, 30 May 2022 12:58:12 +0200 Subject: [PATCH 5/8] Moves qualimap arg to pipeline-specific config --- conf/hasta.config | 4 ---- conf/pipeline/raredisease/hasta.config | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/hasta.config b/conf/hasta.config index 862676a..7e97e5a 100644 --- a/conf/hasta.config +++ b/conf/hasta.config @@ -22,10 +22,6 @@ params { process { executor = 'slurm' clusterOptions = { "-A $params.priority ${params.clusterOptions ?: ''}" } - - withName: QUALIMAP_BAMQC { - ext.args = { "--java-mem-size=${task.memory.giga / 1.15 as long}G" } - } } profiles { diff --git a/conf/pipeline/raredisease/hasta.config b/conf/pipeline/raredisease/hasta.config index 96e3332..a076dbc 100644 --- a/conf/pipeline/raredisease/hasta.config +++ b/conf/pipeline/raredisease/hasta.config @@ -7,4 +7,7 @@ process { cpus = { check_max( 16 * task.attempt, 'cpus' ) } memory = { check_max( 80.GB * task.attempt, 'memory' ) } } + withName:'QUALIMAP_BAMQC' { + ext.args = { "--java-mem-size=${task.memory.giga / 1.15 as long}G" } + } } \ No newline at end of file From 5a67eba04cf0d8850dd2a5b805e3a830cafa9de9 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Fri, 3 Jun 2022 12:41:08 +0200 Subject: [PATCH 6/8] Add MAG @ EVA profile --- README.md | 2 ++ conf/pipeline/mag/eva.config | 14 ++++++++++++++ docs/pipeline/eager/{mpcdf.config => mpcdf.md} | 0 docs/pipeline/mag/eva.md | 15 +++++++++++++++ pipeline/mag.config | 13 +++++++++++++ 5 files changed, 44 insertions(+) create mode 100644 conf/pipeline/mag/eva.config rename docs/pipeline/eager/{mpcdf.config => mpcdf.md} (100%) create mode 100644 docs/pipeline/mag/eva.md create mode 100644 pipeline/mag.config diff --git a/README.md b/README.md index 43b8c71..7224aab 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ Currently documentation is available for the following pipelines within specific - [UPPMAX](docs/pipeline/ampliseq/uppmax.md) - eager - [EVA](docs/pipeline/eager/eva.md) +- mag + - [EVA](docs/pipeline/mag/eva.md) - rnafusion - [MUNIN](docs/pipeline/rnafusion/munin.md) - rnavar diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config new file mode 100644 index 0000000..81d8d0c --- /dev/null +++ b/conf/pipeline/mag/eva.config @@ -0,0 +1,14 @@ +params { + // Specific nf-core/configs params + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' +} + +process { + + withName: FASTQC { + clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } + } + +} diff --git a/docs/pipeline/eager/mpcdf.config b/docs/pipeline/eager/mpcdf.md similarity index 100% rename from docs/pipeline/eager/mpcdf.config rename to docs/pipeline/eager/mpcdf.md diff --git a/docs/pipeline/mag/eva.md b/docs/pipeline/mag/eva.md new file mode 100644 index 0000000..7ba5eec --- /dev/null +++ b/docs/pipeline/mag/eva.md @@ -0,0 +1,15 @@ +# nf-core/configs: eva mag specific configuration + +Extra specific configuration for mag pipeline + +## Usage + +To use, run the pipeline with `-profile eva`. + +This will download and launch the eager specific [`eva.config`](../../../conf/pipeline/mag/eva.config) which has been pre-configured with a setup suitable for the MPI-EVA cluster. + +Example: `nextflow run nf-core/eager -profile eva` + +## mag specific configurations for eva + +Specific configurations for eva has been made for mag, primarily adjusting SGE memory requirements of Java tools (e.g. FastQC). diff --git a/pipeline/mag.config b/pipeline/mag.config new file mode 100644 index 0000000..f45a59a --- /dev/null +++ b/pipeline/mag.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/eager custom profile Nextflow config file + * ------------------------------------------------- + * Config options for custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/mag folder and imported + * under a profile name here. + */ + +profiles { + eva { includeConfig "${params.custom_config_base}/conf/pipeline/mag/eva.config" } +} From 97d57539e3ac9525f2975d827e694840aa98c75f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Jun 2022 12:51:20 +0200 Subject: [PATCH 7/8] Apply suggestions from code review Co-authored-by: Maxime U. Garcia --- docs/pipeline/mag/eva.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/pipeline/mag/eva.md b/docs/pipeline/mag/eva.md index 7ba5eec..7d98117 100644 --- a/docs/pipeline/mag/eva.md +++ b/docs/pipeline/mag/eva.md @@ -6,9 +6,9 @@ Extra specific configuration for mag pipeline To use, run the pipeline with `-profile eva`. -This will download and launch the eager specific [`eva.config`](../../../conf/pipeline/mag/eva.config) which has been pre-configured with a setup suitable for the MPI-EVA cluster. +This will download and launch the mag specific [`eva.config`](../../../conf/pipeline/mag/eva.config) which has been pre-configured with a setup suitable for the MPI-EVA cluster. -Example: `nextflow run nf-core/eager -profile eva` +Example: `nextflow run nf-core/mag -profile eva` ## mag specific configurations for eva From 24e165c52d0ddfbb75517e2497ff9dd20db585cf Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 3 Jun 2022 12:51:47 +0200 Subject: [PATCH 8/8] Update pipeline/mag.config --- pipeline/mag.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/mag.config b/pipeline/mag.config index f45a59a..979392f 100644 --- a/pipeline/mag.config +++ b/pipeline/mag.config @@ -1,6 +1,6 @@ /* * ------------------------------------------------- - * nfcore/eager custom profile Nextflow config file + * nfcore/mag custom profile Nextflow config file * ------------------------------------------------- * Config options for custom environments. * Cluster-specific config options should be saved