From ba8bffbdfdb69fb031bcef52dc64985c680716f2 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 15 Jun 2022 15:39:36 +0200 Subject: [PATCH 1/4] Use env scope instead of beforescript for mag@eva --- conf/pipeline/mag/eva.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config index 81d8d0c..37a531d 100644 --- a/conf/pipeline/mag/eva.config +++ b/conf/pipeline/mag/eva.config @@ -4,6 +4,11 @@ params { config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' } +env { + OPENBLAS_NUM_THREADS=1 + OMP_NUM_THREADS=1 +} + process { withName: FASTQC { From c0effc0366c0dce8815edc733b2143b5da1a19b6 Mon Sep 17 00:00:00 2001 From: James Fellows Yates Date: Wed, 14 Sep 2022 15:34:27 +0200 Subject: [PATCH 2/4] Add HKI profile files --- .github/workflows/main.yml | 1 + README.md | 1 + conf/hki.config | 104 +++++++++++++++++++++++++++++++++++++ docs/hki.md | 24 +++++++++ nfcore_custom.config | 1 + 5 files changed, 131 insertions(+) create mode 100644 conf/hki.config create mode 100644 docs/hki.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca01a15..56755f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -62,6 +62,7 @@ jobs: - "google" - "hasta" - "hebbe" + - "hki" - "icr_davros" - "ifb_core" - "imperial" diff --git a/README.md b/README.md index 4544b05..5264a3c 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Currently documentation is available for the following systems: - [GOOGLE](docs/google.md) - [HASTA](docs/hasta.md) - [HEBBE](docs/hebbe.md) +- [HKI](docs/hki.md) - [ICR_DAVROS](docs/icr_davros.md) - [IMPERIAL](docs/imperial.md) - [JAX](docs/jax.md) diff --git a/conf/hki.config b/conf/hki.config new file mode 100644 index 0000000..b73558f --- /dev/null +++ b/conf/hki.config @@ -0,0 +1,104 @@ +params { + config_profile_description = 'HKI clusters profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' +} + +profiles { + apate { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 128.GB + max_cpus = 32 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + aither { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 128.GB + max_cpus = 32 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + arges { + params { + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_url = 'https://leibniz-hki.de' + max_memory = 64.GB + max_cpus = 12 + max_time = 1440.h + } + process { + executor = 'local' + maxRetries = 2 + } + + executor { + queueSize = 8 + } + + singularity { + enabled = true + autoMounts = true + cacheDir = '/Net/Groups/ccdata/apps/singularity' + } + + conda { + cacheDir = '/Net/Groups/ccdata/apps/conda_envs' + } + + cleanup = true + } + + debug { + cleanup = false + } +} diff --git a/docs/hki.md b/docs/hki.md new file mode 100644 index 0000000..cd102ce --- /dev/null +++ b/docs/hki.md @@ -0,0 +1,24 @@ +# nf-core/configs: HKI Configuration + +All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en/). + +To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluter. The number of parallel jobs that run is currently limited to 8. + +The profiles currently available are: + +- apate (uses singularity, cleanup set to true by default) +- arges (uses singularity, cleanup set to true by default) +- aither (uses singularity, cleanup set to true by default) +- debug (sets cleanup to false for debugging purposes, use e.g. `profile hki,,debug`) + +Note that Nextflow is not necessarily installed by default on the HKI HPC cluster(s). You will need to install it into a directory you have write access to. +Follow these instructions from the Nextflow documentation. + +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + +All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline +has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. + +> NB: You will need an account to use the HKI HPC clusters in order to run the pipeline. If in doubt contact the ICT Service Desk. +> NB: Nextflow will need to submit the jobs via SLURM to the HKI HPC clusters and as such the commands above will have to be executed on the login +> node. If in doubt contact ICT. diff --git a/nfcore_custom.config b/nfcore_custom.config index abf163b..d93e266 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -44,6 +44,7 @@ profiles { google { includeConfig "${params.custom_config_base}/conf/google.config" } hasta { includeConfig "${params.custom_config_base}/conf/hasta.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } + hki { includeConfig "${params.custom_config_base}/conf/hki.config"} icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" } ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" } imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" } From b8751af6c60818102d8a1573016f866572754a5d Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 14 Sep 2022 16:17:17 +0200 Subject: [PATCH 3/4] Update conf/pipeline/mag/eva.config --- conf/pipeline/mag/eva.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conf/pipeline/mag/eva.config b/conf/pipeline/mag/eva.config index 37a531d..81d8d0c 100644 --- a/conf/pipeline/mag/eva.config +++ b/conf/pipeline/mag/eva.config @@ -4,11 +4,6 @@ params { config_profile_description = 'nf-core/mag EVA profile provided by nf-core/configs' } -env { - OPENBLAS_NUM_THREADS=1 - OMP_NUM_THREADS=1 -} - process { withName: FASTQC { From ec638e62dece9fe6519da1a6feb9c1c2f0f517de Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Sep 2022 09:10:29 +0200 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Jasmin F <73216762+jasmezz@users.noreply.github.com> --- conf/hki.config | 6 +++--- docs/hki.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conf/hki.config b/conf/hki.config index b73558f..63718bf 100644 --- a/conf/hki.config +++ b/conf/hki.config @@ -7,7 +7,7 @@ params { profiles { apate { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'apate HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 128.GB @@ -38,7 +38,7 @@ profiles { aither { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'aither HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 128.GB @@ -69,7 +69,7 @@ profiles { arges { params { - config_profile_description = 'apate HKI cluster profile provided by nf-core/configs.' + config_profile_description = 'arges HKI cluster profile provided by nf-core/configs' config_profile_contact = 'James Fellows Yates (@jfy133)' config_profile_url = 'https://leibniz-hki.de' max_memory = 64.GB diff --git a/docs/hki.md b/docs/hki.md index cd102ce..de45c20 100644 --- a/docs/hki.md +++ b/docs/hki.md @@ -1,10 +1,10 @@ # nf-core/configs: HKI Configuration -All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en/). +All nf-core pipelines have been successfully configured for use on clusters at the [Leibniz Institute for Natural Product Research and Infection Biology Hans Knöll Institute](https://www.leibniz-hki.de/en). -To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluter. The number of parallel jobs that run is currently limited to 8. +To use, run the pipeline with `-profile hki,`. This will download and launch the [`hki.config`](../conf/hki.config) which contains specific profiles for each cluster. The number of parallel jobs that run is currently limited to 8. -The profiles currently available are: +The currently available profiles are: - apate (uses singularity, cleanup set to true by default) - arges (uses singularity, cleanup set to true by default)