From 8739aabbf026ebe1655ed0471dacef9ec4c3cbd0 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 2 Feb 2022 08:00:13 +0100 Subject: [PATCH 01/23] Bump markduplicates by default --- conf/pipeline/eager/eva.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 4c2922d..7ba59fa 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -86,6 +86,7 @@ process { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + memory = { check_max( 20.GB * task.attempt, 'memory' ) } } withName: library_merge { @@ -247,6 +248,7 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } } withName: library_merge { @@ -412,6 +414,7 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + memory = { check_max( 48.GB * task.attempt, 'memory' ) } } withName: library_merge { From 7c1c4c7fa441f1ef3c3eb124cab50134f5e7ca1e Mon Sep 17 00:00:00 2001 From: rbpisupati Date: Wed, 2 Feb 2022 22:42:13 +0100 Subject: [PATCH 02/23] changes to cbe module loading --- conf/cbe.config | 4 ++-- docs/cbe.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 0a5763f..e8d7bc0 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -9,11 +9,11 @@ process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } - module = 'anaconda3/2019.10' + module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11'] } singularity { - enabled = true + // enabled = true cacheDir = '/resources/containers' } diff --git a/docs/cbe.md b/docs/cbe.md index aa6c60d..217e1f2 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the CLIP BATCH ENVIRONMENT (CBE) cluster at the Vienna BioCenter (VBC). -To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. This profile sets needed slurm environment. Additionally you need to add `sigularity`, `docker` or `conda` to the profile to be able to run the pipeline. If `-profile cbe,singularity` is given, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to load Nextflow using the environment module system on CBE. You can do this by issuing the commands below: From 50aafc64101de813ac6740c554214202d76fa611 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 3 Feb 2022 17:03:10 +0100 Subject: [PATCH 03/23] Update eva.config --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 7ba59fa..b512921 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -3,7 +3,7 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs' + config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs DEV' } // Specific nf-core/eager process configuration From 86a9042eace4dd244ef22e3054bd0c9889cad4e1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Feb 2022 09:25:01 +0100 Subject: [PATCH 04/23] Update eva.config --- conf/pipeline/eager/eva.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index b512921..c469697 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -134,6 +134,7 @@ process { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } } withName: circularmapper { @@ -301,6 +302,7 @@ profiles { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } } withName: circularmapper { @@ -468,6 +470,7 @@ profiles { withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + memory = { check_max( 32.GB * task.attempt, 'memory' ) } } withName: circularmapper { From 81fe720b00322ea60d6d55b23fe645ada3d0968f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Fri, 4 Feb 2022 09:31:33 +0100 Subject: [PATCH 05/23] Update eva.config --- conf/pipeline/eager/eva.config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index c469697..1530757 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -87,6 +87,7 @@ process { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } memory = { check_max( 20.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1, 143,137, 140] ? 'retry' : 'finish' } } withName: library_merge { @@ -250,6 +251,7 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } memory = { check_max( 32.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [143,137, 140] ? 'retry' : 'finish' } } withName: library_merge { @@ -417,6 +419,7 @@ profiles { withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } memory = { check_max( 48.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [143,137, 140] ? 'retry' : 'finish' } } withName: library_merge { From b5ab4a8258c24001defadf1ea8cc5579020ccee6 Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 4 Feb 2022 10:13:12 +0100 Subject: [PATCH 06/23] Revert "changes to cbe module loading" This reverts commit 7c1c4c7fa441f1ef3c3eb124cab50134f5e7ca1e. --- conf/cbe.config | 4 ++-- docs/cbe.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index e8d7bc0..0a5763f 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -9,11 +9,11 @@ process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } - module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11'] + module = 'anaconda3/2019.10' } singularity { - // enabled = true + enabled = true cacheDir = '/resources/containers' } diff --git a/docs/cbe.md b/docs/cbe.md index 217e1f2..aa6c60d 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -2,7 +2,7 @@ All nf-core pipelines have been successfully configured for use on the CLIP BATCH ENVIRONMENT (CBE) cluster at the Vienna BioCenter (VBC). -To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. This profile sets needed slurm environment. Additionally you need to add `sigularity`, `docker` or `conda` to the profile to be able to run the pipeline. If `-profile cbe,singularity` is given, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to load Nextflow using the environment module system on CBE. You can do this by issuing the commands below: From 52b6f7da211d1fc123cf100a340c9b45564e88e2 Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 4 Feb 2022 10:18:59 +0100 Subject: [PATCH 07/23] update anaconda module not sure why a binary distribution needs a buildenv at all, but well --- conf/cbe.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cbe.config b/conf/cbe.config index 89da3db..a8bf05d 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -8,7 +8,7 @@ params { process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } - module = 'anaconda3/2019.10' + module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11'] // --signal option will be handled by nextflow after 21.10.0 release (see https://github.com/nextflow-io/nextflow/issues/2163) clusterOptions = { '--signal B:USR2 ' << ( (queue == 'c' & task.time <= 1.h) ? '--qos rapid' : ( task.time <= 8.h ? '--qos short': ( task.time <= 48.h ? '--qos medium' : '--qos long' ) ) ) } From a6dd3aa0f1fedba102d4e1ef1f587671e5e6331c Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 4 Feb 2022 10:21:52 +0100 Subject: [PATCH 08/23] cbe: update recommended nextflow version in docs otherwise, the new anaconda requirements will cause conflicts --- docs/cbe.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cbe.md b/docs/cbe.md index aa6c60d..b7628d0 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -9,7 +9,7 @@ Before running the pipeline you will need to load Nextflow using the environment ```bash ## Load Nextflow environment module module purge -module load nextflow/19.04.0 +module load nextflow/21.10.6 ``` A local copy of the [AWS-iGenomes](https://registry.opendata.aws/aws-igenomes/) resource has been made available on CBE so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. From 49810a69b399c32c52bf96fa20813c049b646922 Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 4 Feb 2022 10:31:58 +0100 Subject: [PATCH 09/23] cbe: add gpu support just in case.. note that this will need --nv in containerOptions in order to mount cuda libs into the container --- conf/cbe.config | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index a8bf05d..1cdccba 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -9,9 +9,7 @@ process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11'] - - // --signal option will be handled by nextflow after 21.10.0 release (see https://github.com/nextflow-io/nextflow/issues/2163) - clusterOptions = { '--signal B:USR2 ' << ( (queue == 'c' & task.time <= 1.h) ? '--qos rapid' : ( task.time <= 8.h ? '--qos short': ( task.time <= 48.h ? '--qos medium' : '--qos long' ) ) ) } + clusterOptions = { ( task.queue == 'g' ? '--gres gpu:1 ' : '' ) << ( (task.queue == 'c' & task.time <= 1.h) ? '--qos rapid' : ( task.time <= 8.h ? '--qos short': ( task.time <= 48.h ? '--qos medium' : '--qos long' ) ) ) } } singularity { From 7bf6f284fac8b90dc674fb766c3b0bf03943b764 Mon Sep 17 00:00:00 2001 From: phue Date: Fri, 4 Feb 2022 10:33:00 +0100 Subject: [PATCH 10/23] cbe: lower the threshold for switching to m queue --- conf/cbe.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cbe.config b/conf/cbe.config index 1cdccba..fbc0812 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -7,7 +7,7 @@ params { process { executor = 'slurm' - queue = { task.memory <= 170.GB ? 'c' : 'm' } + queue = { task.memory <= 120.GB ? 'c' : 'm' } module = ['build-env/.f2021', 'build-env/f2021', 'anaconda3/2021.11'] clusterOptions = { ( task.queue == 'g' ? '--gres gpu:1 ' : '' ) << ( (task.queue == 'c' & task.time <= 1.h) ? '--qos rapid' : ( task.time <= 8.h ? '--qos short': ( task.time <= 48.h ? '--qos medium' : '--qos long' ) ) ) } } From dbb71161c5428e77d8c4f6f8e347fa4b269fb139 Mon Sep 17 00:00:00 2001 From: Stevin Wilson Date: Fri, 4 Feb 2022 15:31:51 -0500 Subject: [PATCH 11/23] Added ARTIC v4.1 primer set information --- conf/pipeline/viralrecon/genomes.config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conf/pipeline/viralrecon/genomes.config b/conf/pipeline/viralrecon/genomes.config index 8682942..baad834 100644 --- a/conf/pipeline/viralrecon/genomes.config +++ b/conf/pipeline/viralrecon/genomes.config @@ -51,6 +51,12 @@ params { primer_bed = 'https://github.com/artic-network/artic-ncov2019/raw/master/primer_schemes/nCoV-2019/V4/SARS-CoV-2.scheme.bed' scheme = 'SARS-CoV-2' } + '4.1' { + fasta = 'https://github.com/artic-network/artic-ncov2019/raw/master/primer_schemes/nCoV-2019/V4.1/SARS-CoV-2.reference.fasta' + gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' + primer_bed = 'https://github.com/artic-network/artic-ncov2019/raw/master/primer_schemes/nCoV-2019/V4.1/SARS-CoV-2.scheme.bed' + scheme = 'SARS-CoV-2' + } '1200' { fasta = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/primer_schemes/artic/nCoV-2019/V1200/nCoV-2019.reference.fasta' gff = 'https://github.com/nf-core/test-datasets/raw/viralrecon/genome/MN908947.3/GCA_009858895.3_ASM985889v3_genomic.200409.gff.gz' From aa525c6ebf6d28da67fc442d9d5d2b50d8b0be01 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 6 Feb 2022 09:07:53 +0100 Subject: [PATCH 12/23] Allow exit code 1 retry for all java processes --- conf/pipeline/eager/eva.config | 51 ++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 1530757..7b7cef0 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -62,14 +62,17 @@ process { withName: makeSeqDict { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: fastqc { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: adapter_removal { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: samtools_flagstat { @@ -82,12 +85,13 @@ process { withName: dedup { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } memory = { check_max( 20.GB * task.attempt, 'memory' ) } - errorStrategy = { task.exitStatus in [1, 143,137, 140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: library_merge { @@ -115,35 +119,43 @@ process { withName: maltextract { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: multivcfanalyzer { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: mtnucratio { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: vcf2genome { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } - memory = { check_max( 8.GB * task.attempt, 'memory' ) } + memory = { check_max( 8.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circularmapper { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circulargenerator { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: preseq { @@ -153,6 +165,7 @@ process { withName: genotyping_ug { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } } @@ -226,14 +239,17 @@ profiles { withName: makeSeqDict { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: fastqc { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: adapter_removal { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: samtools_flagstat { @@ -246,12 +262,13 @@ profiles { withName: dedup { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } memory = { check_max( 32.GB * task.attempt, 'memory' ) } - errorStrategy = { task.exitStatus in [143,137, 140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: library_merge { @@ -275,6 +292,7 @@ profiles { clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G,virtual_free=1000G" } cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 955.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName:hostremoval_input_fastq { @@ -284,39 +302,48 @@ profiles { withName: maltextract { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: multivcfanalyzer { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: mtnucratio { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: vcf2genome { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circularmapper { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circulargenerator { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: genotyping_ug { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: preseq { @@ -394,14 +421,17 @@ profiles { withName: makeSeqDict { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: fastqc { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: adapter_removal { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: samtools_flagstat { @@ -414,12 +444,13 @@ profiles { withName: dedup { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: markduplicates { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } memory = { check_max( 48.GB * task.attempt, 'memory' ) } - errorStrategy = { task.exitStatus in [143,137, 140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: library_merge { @@ -449,43 +480,53 @@ profiles { clusterOptions = { "-S /bin/bash -V -l h_vmem=1000G,virtual_free=1000G" } cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 955.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: maltextract { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: multivcfanalyzer { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: mtnucratio { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: vcf2genome { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: damageprofiler { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } memory = { check_max( 32.GB * task.attempt, 'memory' ) } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circularmapper { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: circulargenerator { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: genotyping_ug { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } } withName: preseq { From 8c5a54ac9ff8696c19afe252d774a445ff1c4987 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Sun, 6 Feb 2022 19:37:53 +0100 Subject: [PATCH 13/23] Update conf/pipeline/eager/eva.config --- conf/pipeline/eager/eva.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 7b7cef0..d13a71a 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -3,7 +3,7 @@ params { // Specific nf-core/configs params config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs DEV' + config_profile_description = 'nf-core/eager EVA profile provided by nf-core/configs' } // Specific nf-core/eager process configuration From 2b06293f64b8f81fe4ae8799e13d38ab5bcd7f67 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 8 Feb 2022 11:15:17 +0100 Subject: [PATCH 14/23] Read add empty BAM exit code for qualitmap 255 --- conf/pipeline/eager/eva.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index d13a71a..d436895 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -139,7 +139,7 @@ process { withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" } - errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' } } withName: damageprofiler { @@ -322,7 +322,7 @@ profiles { withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" } - errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' } } withName: damageprofiler { @@ -505,7 +505,7 @@ profiles { withName: qualimap { clusterOptions = { "-S /bin/bash -V -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" } - errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in [1,143,137,104,134,139,140] ? 'retry' : task.exitStatus in [255] ? 'ignore' : 'finish' } } withName: damageprofiler { From fa4075c2a0302995efdb0558ddafcbbe0b82ecfa Mon Sep 17 00:00:00 2001 From: "Thiseas C. Lamnidis" Date: Thu, 10 Feb 2022 10:00:18 +0100 Subject: [PATCH 15/23] Removed all time limits --- conf/pipeline/eager/eva.config | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index d436895..67c3b5a 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,43 +19,36 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use @@ -196,43 +189,36 @@ profiles { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.5.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 24.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } // Fixes for SGE and Java incompatibility due to (and also some samtools?!) using more memory than you tell it to use @@ -297,7 +283,6 @@ profiles { withName:hostremoval_input_fastq { memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = 1440.h } withName: maltextract { @@ -378,43 +363,36 @@ profiles { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 2.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 512.GB * task.attempt, 'memory' ) } - time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use @@ -469,7 +447,6 @@ profiles { withName:hostremoval_input_fastq { memory = { check_max( 32.GB * task.attempt, 'memory' ) } - time = 1440.h } withName: metagenomic_complexity_filter { From c0ee3225ad976f9f25e73948fbfa2fb13a7d9d88 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:02:52 +0100 Subject: [PATCH 16/23] Add times for 365d --- conf/pipeline/eager/eva.config | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 67c3b5a..e6f673e 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,36 +19,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = '365.d' } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use From 39c2d9b25b96e49bc58672a3ff3fedccc443edf2 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:19:16 +0100 Subject: [PATCH 17/23] Oops --- conf/pipeline/eager/eva.config | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index e6f673e..12bb4da 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,43 +19,42 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = '365.d' + time = '365d' } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = '365.d' + time = '365d' } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = '365.d' + time = '365d' } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = '365.d' + time = '365d' } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = '365.d' + time = '365d' } withLabel:'mc_large'{ - cpus = { check_max( 8, 'cpus' ) } - memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = '365.d' + cpus = { check_max( 8, 'cpus' ) } + memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = '365d' } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } - time = '365.d' } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use From dd94895f168f2764f6580ca6a0f468af478ce2c3 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:22:45 +0100 Subject: [PATCH 18/23] remove quotes --- conf/pipeline/eager/eva.config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 12bb4da..64138af 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,37 +19,37 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = '365d' + time = 365d } withLabel:'mc_huge'{ From dbd78f8075c9d4a3483cd3da23a4a38b1ac97af6 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:26:18 +0100 Subject: [PATCH 19/23] Re add `.` --- conf/pipeline/eager/eva.config | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 64138af..669097d 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,37 +19,37 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = 365d + time = 365.d } withLabel:'mc_huge'{ From d7d0f5ceccea4d28ceab5c5e4c1fa75642955073 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:32:49 +0100 Subject: [PATCH 20/23] Change base time --- conf/eva.config | 2 +- conf/pipeline/eager/eva.config | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/conf/eva.config b/conf/eva.config index 8b52dda..432b667 100644 --- a/conf/eva.config +++ b/conf/eva.config @@ -30,7 +30,7 @@ profiles { config_profile_description = 'MPI-EVA archgen profile, provided by nf-core/configs.' max_memory = 256.GB max_cpus = 32 - max_time = 720.h + max_time = 365.d //Illumina iGenomes reference file path } diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 669097d..67c3b5a 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,37 +19,31 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } - time = 365.d } withLabel:'mc_huge'{ From 88ddc9eee7a158200cd5e2423ad5d721c4eeaebe Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Thu, 10 Feb 2022 11:34:57 +0100 Subject: [PATCH 21/23] fffff --- conf/pipeline/eager/eva.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/pipeline/eager/eva.config b/conf/pipeline/eager/eva.config index 67c3b5a..eccccbb 100644 --- a/conf/pipeline/eager/eva.config +++ b/conf/pipeline/eager/eva.config @@ -19,36 +19,43 @@ process { withLabel:'sc_tiny'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 1.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'sc_small'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'sc_medium'{ cpus = { check_max( 1, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_small'{ cpus = { check_max( 2, 'cpus' ) } memory = { check_max( 4.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_medium' { cpus = { check_max( 4, 'cpus' ) } memory = { check_max( 8.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_large'{ cpus = { check_max( 8, 'cpus' ) } memory = { check_max( 16.GB * task.attempt, 'memory' ) } + time = '365.d' } withLabel:'mc_huge'{ cpus = { check_max( 32, 'cpus' ) } memory = { check_max( 256.GB * task.attempt, 'memory' ) } + time = '365.d' } // Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use From fcc856c8da5f76446b5fba182061008566046f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 11 Feb 2022 11:03:18 +0000 Subject: [PATCH 22/23] update node version for markdownlint --- .github/workflows/linting.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index f6a2458..0fa938a 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -7,9 +7,7 @@ jobs: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 - with: - node-version: '10' + - uses: actions/setup-node@v2 - name: Install markdownlint run: | npm install -g markdownlint-cli From b98ab26662f3683a898078d235bccc782f3178b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 11 Feb 2022 14:09:16 +0100 Subject: [PATCH 23/23] fix markdownlint errors --- docs/czbiohub.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/czbiohub.md b/docs/czbiohub.md index 03f0edd..376460b 100644 --- a/docs/czbiohub.md +++ b/docs/czbiohub.md @@ -125,7 +125,7 @@ For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annota ## High Priority Queue -If you would like to run with the _High Priority_ queue, specify the `highpriority` config profile after `czbiohub_aws`. When applied after the main `czbiohub_aws` config, it overwrites the process `queue` identifier. +If you would like to run with the *High Priority* queue, specify the `highpriority` config profile after `czbiohub_aws`. When applied after the main `czbiohub_aws` config, it overwrites the process `queue` identifier. To use it, submit your run with with `-profile czbiohub_aws,highpriority`.