From 1c7230cb08791b93bcca612836b5d68afade8b66 Mon Sep 17 00:00:00 2001 From: Bruno Grande Date: Thu, 23 Jun 2022 09:15:42 -0700 Subject: [PATCH] Add Sage-specific Sarek config --- README.md | 2 +- conf/pipeline/sarek/sage.config | 37 +++++++++++++++++++++++++++++++++ docs/pipeline/sarek/sage.md | 7 +++++++ docs/sage.md | 14 +++++++------ pipeline/sarek.config | 3 ++- 5 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 conf/pipeline/sarek/sage.config create mode 100644 docs/pipeline/sarek/sage.md diff --git a/README.md b/README.md index 978e606..0d561fc 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ A repository for hosting Nextflow configuration files containing custom paramete - [Configuration and parameters](#configuration-and-parameters) - [Offline usage](#offline-usage) - [Adding a new config](#adding-a-new-config) - - [Checking user hostnames](#checking-user-hostnames) - [Testing](#testing) - [Documentation](#documentation) - [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) @@ -203,6 +202,7 @@ Currently documentation is available for the following pipelines within specific - [MUNIN](docs/pipeline/rnavar/munin.md) - sarek - [MUNIN](docs/pipeline/sarek/munin.md) + - [SAGE BIONETWORKS](docs/pipeline/sarek/sage.md) - [UPPMAX](docs/pipeline/sarek/uppmax.md) - taxprofiler - [EVA](docs/pipeline/taxprofiler/eva.md) diff --git a/conf/pipeline/sarek/sage.config b/conf/pipeline/sarek/sage.config new file mode 100644 index 0000000..b7766bc --- /dev/null +++ b/conf/pipeline/sarek/sage.config @@ -0,0 +1,37 @@ +params { + config_profile_description = 'The Sage Bionetworks profile' + config_profile_contact = 'Bruno Grande (@BrunoGrandePhD)' + config_profile_url = 'https://github.com/Sage-Bionetworks-Workflows' +} + +// Function to ensure that resource requirements don't go beyond a maximum limit +def check_max(obj, type) { + if (type == 'memory') { + try { + if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) + return params.max_memory as nextflow.util.MemoryUnit + else + return obj + } catch (all) { + println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'time') { + try { + if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) + return params.max_time as nextflow.util.Duration + else + return obj + } catch (all) { + println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" + return obj + } + } else if (type == 'cpus') { + try { + return Math.min( obj, params.max_cpus as int ) + } catch (all) { + println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" + return obj + } + } +} diff --git a/docs/pipeline/sarek/sage.md b/docs/pipeline/sarek/sage.md new file mode 100644 index 0000000..646fdda --- /dev/null +++ b/docs/pipeline/sarek/sage.md @@ -0,0 +1,7 @@ +# nf-core/configs: Sage Bionetworks Sarek-Specific Configuration + +To use this custom configuration, run the pipeline with `-profile sage`. This will download and load the [`sage.config`](../conf/sage.config), which contains a number of optimizations relevant to Sage employees running workflows on AWS (_e.g._ using Nextflow Tower). This profile will also load any applicable pipeline-specific configuration. + +In addition to the global configuration described [here](../../sage.md), this Sarek-specific configuration includes the following tweaks: + +- Define the `check_max()` function, which is missing in Sarek v2. diff --git a/docs/sage.md b/docs/sage.md index 755e0c2..5b0fa49 100644 --- a/docs/sage.md +++ b/docs/sage.md @@ -1,12 +1,14 @@ -# nf-core/configs: Sage Bionetworks Configuration +# nf-core/configs: Sage Bionetworks Global Configuration -To use this custom configuration, run the pipeline with `-profile sage`. This will download and launch the [`sage.config`](../conf/sage.config), which contains a number of optimizations relevant to Sage employees running workflows on AWS (_e.g._ using Nextflow Tower). These include: +To use this custom configuration, run the pipeline with `-profile sage`. This will download and load the [`sage.config`](../conf/sage.config), which contains a number of optimizations relevant to Sage employees running workflows on AWS (_e.g._ using Nextflow Tower). This profile will also load any applicable pipeline-specific configuration. -- Updating the default value for `igenomes_base` to `s3://sage-igenomes` -- Increasing the default time limits because we run pipelines on AWS -- Enabling retries by default when exit codes relate to insufficient memory +This global configuration includes the following tweaks: + +- Update the default value for `igenomes_base` to `s3://sage-igenomes` +- Increase the default time limits because we run pipelines on AWS +- Enable retries by default when exit codes relate to insufficient memory - Allow pending jobs to finish if the number of retries are exhausted -- Slowing the increase in the number of allocated CPU cores on retries +- Slow the increase in the number of allocated CPU cores on retries ## Additional information about iGenomes diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 57d7bdf..36efc99 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -15,4 +15,5 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" } cfc_dev { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" } eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" } -} \ No newline at end of file + sage { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/sage.config" } +}