From cd1d0b5f8f52ced05548cbd3b6dddd680cf890e7 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 10:10:05 +0100 Subject: [PATCH 01/32] Add specific sarek config base, munin specific sarek config and docs rename file improve configs remove unnecessary params --- conf/munin.config | 26 +++++++++++--------------- conf/pipeline/sarek/munin.config | 26 ++++++++++++++++++++++++++ docs/pipeline/sarek/munin.md | 21 +++++++++++++++++++++ pipeline/sarek.config | 13 +++++++++++++ 4 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 conf/pipeline/sarek/munin.config create mode 100644 docs/pipeline/sarek/munin.md create mode 100644 pipeline/sarek.config diff --git a/conf/munin.config b/conf/munin.config index f53def4..e642ec6 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -1,8 +1,13 @@ -//Profile config names for nf-core/configs +// Profile config names for nf-core/configs + params { - config_profile_description = 'MUNIN profile provided by nf-core/configs.' + // Specific nf-core/configs params config_profile_contact = 'Szilveszter Juhos (@szilva)' - config_profile_url = '' + config_profile_description = 'MUNIN profile provided by nf-core/configs.' + config_profile_url = 'https://ki.se/forskning/barntumorbanken' + + // Local AWS iGenomes reference file paths on munin + igenomes_base = '/data1/references/igenomes/' } process { @@ -10,9 +15,10 @@ process { maxForks = 46 } -// To use singularity, use nextflow run -profile munin,singularity +// To use singularity, use nextflow run -profile munin singularity { enabled = true + cacheDir = '/data1/containers/' } // To use docker, use nextflow run -profile munin,docker @@ -20,14 +26,4 @@ docker { enabled = false mountFlags = 'z' fixOwnership = true -} - -params { - // general params - max_memory = 752.GB - max_cpus = 46 - max_time = 72.h - - // Local AWS iGenomes reference file paths on munin - igenomes_base = '/data1/references/igenomes/' -} +} \ No newline at end of file diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config new file mode 100644 index 0000000..8655a2f --- /dev/null +++ b/conf/pipeline/sarek/munin.config @@ -0,0 +1,26 @@ +// Profile config names for nf-core/configs + +params { + // Specific nf-core/configs params + config_profile_contact = 'Maxime Garcia (@MaxUlysse)' + config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' + config_profile_url = 'https://ki.se/forskning/barntumorbanken' + + // Specific nf-core/sarek configs params + max_memory = 752.GB + max_cpus = 46 + max_time = 72.h + + // Specific nf-core/sarek params + annotation_cache = true + pon = '/data1/PON/vcfs/BTB.PON.vcf.gz' + pon_index = '/data1/PON/vcfs/BTB.PON.vcf.gz.tbi' + snpEff_cache = '/data1/cache/snpEff/' + vep_cache = '/data1/cache/VEP/' +} + +// Specific nf-core/sarek configuration +withLabel:sentieon { + module = {params.sentieon ? 'sentieon/201808.05' : ''} + container = {params.sentieon ? '' : 'nfcore/sarek:dev'} +} diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md new file mode 100644 index 0000000..24c4404 --- /dev/null +++ b/docs/pipeline/sarek/munin.md @@ -0,0 +1,21 @@ +# nf-core/configs: MUNIN sarek specific configuration + +Extra specific configuration for sarek pipeline + +## Usage + +To use, run the pipeline with `-profile munin`. + +This will download and launch the [`munin.config`](../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. + +Example: `nextflow run nf-core/sarek -profile munin` + +## Sarek specific configurations for Munin + +Specific configurations for Munin has been made for sarek. + +* Path to `snpEff_cache`: `/data1/cache/snpEff/` +* Path to `vep_cache`: `/data1/cache/VEP/` +* Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` +* Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` +* Load module `Sentieon` for Process with `sentieon` labels \ No newline at end of file diff --git a/pipeline/sarek.config b/pipeline/sarek.config new file mode 100644 index 0000000..6935794 --- /dev/null +++ b/pipeline/sarek.config @@ -0,0 +1,13 @@ +/* + * ------------------------------------------------- + * nfcore/sarek custom profile Nextflow config file + * ------------------------------------------------- + * Config options for all custom environments. + * Cluster-specific config options should be saved + * in the conf/pipeline/sarek folder and imported + * under a profile name here. + */ + +profiles { + munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } +} \ No newline at end of file From 25f6ab6dd9d29144f620ce088fdf466ced11e771 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 14:34:06 +0100 Subject: [PATCH 02/32] add docs --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index b9812b6..9e40d67 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,50 @@ Commit and push these changes to your local clone on GitHub, and then [create a We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. +## Adding a new pipeline specific config + +Follow the previous guidelines to create a nf-core/configs, update the pipeline specific documentation, and create the necessary files. +Replace `` with the pipeline name for all the instructions in this guidelines. +In a similar manner, replace `` by the name of the profile. + +### Pipeline specific documentation + +Currently documentation is available for the following pipelines within the specific profile: + +* sarek + * [MUNIN](docs/pipeline/sarek/munin.md) + +### Enabling the specific configs within a specific PIPELINE + +> This has to be done on a fork of the `nf-core/` repository. + +If not already present, add to `nextflow.config` + +```Groovy +// Load nf-core/ custom profiles from different Institutions +try { + includeConfig "${params.custom_config_base}/pipeline/.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/ profiles: ${params.custom_config_base}/pipeline/.config") +} +``` + +### Create the specific nf-core/configs for the pipeline + +> This has to be done on a fork of the `nf-core/configs` repository. + +If not already created, add the `pipeline/.config` file, and add the profile to the profile scope + +```Groovy +profiles { + { includeConfig "${params.custom_config_base}/conf/pipeline//.config" } +} +``` + +Add the `conf/pipeline//.config` file with all pipeline specific params, `withLabel` or any other configuration profile. + +Add the `docs/pipeline//.md` file with documentation on what are the specificity of this configs. + ## Help If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). From 43f6f55ba2999cf323bcf94f286fcd2fc460236a Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:13:13 +0100 Subject: [PATCH 03/32] Update pipeline/sarek.config --- pipeline/sarek.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 6935794..aed6a1d 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -2,7 +2,7 @@ * ------------------------------------------------- * nfcore/sarek custom profile Nextflow config file * ------------------------------------------------- - * Config options for all custom environments. + * Config options for custom environments. * Cluster-specific config options should be saved * in the conf/pipeline/sarek folder and imported * under a profile name here. @@ -10,4 +10,4 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } -} \ No newline at end of file +} From 283abcece42074eead291e67166dbdc0b2d3ffba Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 15:32:35 +0100 Subject: [PATCH 04/32] update docs --- README.md | 6 +++++- docs/pipeline/sarek/munin.md | 7 ++++--- pipeline/sarek.config | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9e40d67..06bd456 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,10 @@ A repository for hosting nextflow config files containing custom parameters requ * [Testing](#testing) * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Adding a new pipeline specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline Specific Documentation](#pipeline-specific-documentation) + * [Enabling the specific configs within a specific pipeline](#enabling-the-specific-configs-within-a-specific-pipeline) + * [Create the specific `nf-core/configs` for the pipeline](#create-the-specific-nf-coreconfigs-for-the-pipeline) * [Help](#help) ## Using an existing config @@ -128,7 +132,7 @@ Currently documentation is available for the following pipelines within the spec * sarek * [MUNIN](docs/pipeline/sarek/munin.md) -### Enabling the specific configs within a specific PIPELINE +### Enabling the specific configs within a specific pipeline > This has to be done on a fork of the `nf-core/` repository. diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index 24c4404..cd93648 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -10,12 +10,13 @@ This will download and launch the [`munin.config`](../conf/pipeline/sarek/munin. Example: `nextflow run nf-core/sarek -profile munin` -## Sarek specific configurations for Munin +## Sarek specific configurations for MUNIN -Specific configurations for Munin has been made for sarek. +Specific configurations for MUNIN has been made for sarek. +* Params `annotation_cache` set to `true` * Path to `snpEff_cache`: `/data1/cache/snpEff/` * Path to `vep_cache`: `/data1/cache/VEP/` * Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` * Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` -* Load module `Sentieon` for Process with `sentieon` labels \ No newline at end of file +* Load module `Sentieon` for Processes with `sentieon` labels \ No newline at end of file diff --git a/pipeline/sarek.config b/pipeline/sarek.config index aed6a1d..9457972 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -10,4 +10,4 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } -} +} \ No newline at end of file From 73047968cf8e8a171b75491f4821085adb115ed7 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:53:49 +0100 Subject: [PATCH 05/32] Update conf/pipeline/sarek/munin.config --- conf/pipeline/sarek/munin.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 8655a2f..9ccc713 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -4,7 +4,6 @@ params { // Specific nf-core/configs params config_profile_contact = 'Maxime Garcia (@MaxUlysse)' config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' - config_profile_url = 'https://ki.se/forskning/barntumorbanken' // Specific nf-core/sarek configs params max_memory = 752.GB From b029ccaa59c38bd1e1fe5e6be53e0bb0a7cf6aef Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:54:29 +0100 Subject: [PATCH 06/32] Update docs/pipeline/sarek/munin.md Update conf/pipeline/sarek/munin.config --- conf/pipeline/sarek/munin.config | 1 - docs/pipeline/sarek/munin.md | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 9ccc713..69c16eb 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -21,5 +21,4 @@ params { // Specific nf-core/sarek configuration withLabel:sentieon { module = {params.sentieon ? 'sentieon/201808.05' : ''} - container = {params.sentieon ? '' : 'nfcore/sarek:dev'} } diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index cd93648..ce411c1 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -6,7 +6,7 @@ Extra specific configuration for sarek pipeline To use, run the pipeline with `-profile munin`. -This will download and launch the [`munin.config`](../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. +This will download and launch the sarek specific [`munin.config`](../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. Example: `nextflow run nf-core/sarek -profile munin` @@ -19,4 +19,4 @@ Specific configurations for MUNIN has been made for sarek. * Path to `vep_cache`: `/data1/cache/VEP/` * Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` * Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` -* Load module `Sentieon` for Processes with `sentieon` labels \ No newline at end of file +* Load module `Sentieon` for Processes with `sentieon` labels From 9a57e487d76ee415d04bf48b768863fc047c2db0 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 16:00:05 +0100 Subject: [PATCH 07/32] Apply suggestions from code review --- conf/pipeline/sarek/munin.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 69c16eb..eccd147 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -21,4 +21,5 @@ params { // Specific nf-core/sarek configuration withLabel:sentieon { module = {params.sentieon ? 'sentieon/201808.05' : ''} + if (params.sentieon) container = '' } From edde1ee64dffd02a922a78c1fba54f0a74fb4281 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 16:03:27 +0100 Subject: [PATCH 08/32] fix container/module collision --- conf/pipeline/sarek/munin.config | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index eccd147..b7b92f2 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -18,8 +18,10 @@ params { vep_cache = '/data1/cache/VEP/' } -// Specific nf-core/sarek configuration -withLabel:sentieon { - module = {params.sentieon ? 'sentieon/201808.05' : ''} - if (params.sentieon) container = '' -} +// Specific nf-core/sarek process configuration +process { + withLabel:sentieon { + beforeScript = {params.sentieon ? 'module load sentieon/201808.05' : ''} + container = {params.sentieon ? '' : 'nfcore/sarek:dev'} + } +} \ No newline at end of file From e98f74e47b1e2d035825efe2d99e4d4882088d84 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 21 Nov 2019 14:21:32 +0100 Subject: [PATCH 09/32] actually belong to nf-core/configs/munin --- conf/munin.config | 5 +++++ conf/pipeline/sarek/munin.config | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index e642ec6..9551b91 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -8,6 +8,11 @@ params { // Local AWS iGenomes reference file paths on munin igenomes_base = '/data1/references/igenomes/' + + // General cpus/memory/time requirements + max_cpus = 46 + max_memory = 752.GB + max_time = 72.h } process { diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index b7b92f2..d091a4c 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -5,11 +5,6 @@ params { config_profile_contact = 'Maxime Garcia (@MaxUlysse)' config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' - // Specific nf-core/sarek configs params - max_memory = 752.GB - max_cpus = 46 - max_time = 72.h - // Specific nf-core/sarek params annotation_cache = true pon = '/data1/PON/vcfs/BTB.PON.vcf.gz' From 29f0442c3a4c0c25fae1dd248f75e361291e3478 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 21 Nov 2019 15:25:12 +0100 Subject: [PATCH 10/32] Update README.md Co-Authored-By: James A. Fellows Yates --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 06bd456..a6b0cd1 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ Currently documentation is available for the following pipelines within the spec ### Enabling the specific configs within a specific pipeline -> This has to be done on a fork of the `nf-core/` repository. +:warning: **This has to be done on a fork of the `nf-core/` repository.** If not already present, add to `nextflow.config` @@ -149,7 +149,7 @@ try { ### Create the specific nf-core/configs for the pipeline -> This has to be done on a fork of the `nf-core/configs` repository. +:warning: This has to be done on a fork of the `nf-core/configs` repository. If not already created, add the `pipeline/.config` file, and add the profile to the profile scope From 93d5156c7dab6f08e1c98d4a7225a13d494caf44 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Fri, 22 Nov 2019 13:55:08 +0100 Subject: [PATCH 11/32] Update conf/munin.config --- conf/munin.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 9551b91..df45787 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -2,7 +2,7 @@ params { // Specific nf-core/configs params - config_profile_contact = 'Szilveszter Juhos (@szilva)' + config_profile_contact = 'Szilveszter Juhos (@szilvajuhos)' config_profile_description = 'MUNIN profile provided by nf-core/configs.' config_profile_url = 'https://ki.se/forskning/barntumorbanken' @@ -31,4 +31,4 @@ docker { enabled = false mountFlags = 'z' fixOwnership = true -} \ No newline at end of file +} From 028008c1d0867927efb76acf6f18ce020e255c17 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 22 Nov 2019 14:42:40 +0100 Subject: [PATCH 12/32] improve syntax --- conf/pipeline/sarek/munin.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index d091a4c..b2670a9 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -16,7 +16,9 @@ params { // Specific nf-core/sarek process configuration process { withLabel:sentieon { - beforeScript = {params.sentieon ? 'module load sentieon/201808.05' : ''} - container = {params.sentieon ? '' : 'nfcore/sarek:dev'} + if (params.sentieon) { + module = 'sentieon/201808.05' + container = '' + } } } \ No newline at end of file From 209763620a53b532183d08a6f2fc8bec60e7a44a Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 10:30:24 +0100 Subject: [PATCH 13/32] update README --- README.md | 92 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index a6b0cd1..3f42311 100644 --- a/README.md +++ b/README.md @@ -5,35 +5,40 @@ A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents + * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) * [Adding a new config](#adding-a-new-config) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) * [Adding a new pipeline specific config](#adding-a-new-pipeline-specific-config) - * [Pipeline Specific Documentation](#pipeline-specific-documentation) - * [Enabling the specific configs within a specific pipeline](#enabling-the-specific-configs-within-a-specific-pipeline) - * [Create the specific `nf-core/configs` for the pipeline](#create-the-specific-nf-coreconfigs-for-the-pipeline) + * [Pipeline Specific Documentation](#pipeline-specific-documentation) + * [Enabling the specific configs within a specific pipeline](#enabling-the-specific-configs-within-a-specific-pipeline) + * [Create the specific `nf-core/configs` for the pipeline](#create-the-specific-nf-coreconfigs-for-the-pipeline) * [Help](#help) ## Using an existing config -The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. +The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. +However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. ### Configuration and parameters The config files hosted in this repository define a set of parameters which are specific to compute environments at different Institutions but generic enough to be used with all nf-core pipelines. -All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. +All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. +For example, if you have the ability to use [Singularity](https://sylabs.io/singularity/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. +Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. +In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. You should be able to get a good idea as to how other people are customising the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). ### Offline usage -If you want to use an existing config available in `nf-core/configs`, and you're running on a system that has no internet connection, you'll need to download the config file and place it in a location that is visible to the file system on which you are running the pipeline. Then run the pipeline with `--custom_config_base` -or `params.custom_config_base` set to the location of the directory containing the repository files: +If you want to use an existing config available in `nf-core/configs`, and you're running on a system that has no internet connection, you'll need to download the config file and place it in a location that is visible to the file system on which you are running the pipeline. +Then run the pipeline with `--custom_config_base` or `params.custom_config_base` set to the location of the directory containing the repository files: ```bash ## Download and unzip the config files @@ -46,28 +51,28 @@ cd /path/to/my/data nextflow run /path/to/pipeline/ --custom_config_base /path/to/my/configs/configs-master/ ``` -Alternatively, instead of using the configuration profiles from this repository, you can run your -pipeline directly calling the single institutional config file that you need with the `-c` parameter. +Alternatively, instead of using the configuration profiles from this repository, you can run your pipeline directly calling the single institutional config file that you need with the `-c` parameter. ```bash nextflow run /path/to/pipeline/ -c /path/to/my/configs/configs-master/conf/my_config.config ``` -> Note that the nf-core/tools helper package has a `download` command to download all required pipeline -> files + singularity containers + institutional configs in one go for you, to make this process easier. +> Note that the nf-core/tools helper package has a `download` command to download all required pipeline files + singularity containers + institutional configs in one go for you, to make this process easier. ## Adding a new config -If you decide to upload your custom config file to `nf-core/configs` then this will ensure that your custom config file will be automatically downloaded, and available at run-time to all nf-core pipelines, and to everyone within your organisation. You will simply have to specify `-profile ` in the command used to run the pipeline. See [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. +If you decide to upload your custom config file to `nf-core/configs` then this will ensure that your custom config file will be automatically downloaded, and available at run-time to all nf-core pipelines, and to everyone within your organisation. +You will simply have to specify `-profile ` in the command used to run the pipeline. +See [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for examples. -Please also make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` and `params.config_profile_url` set to reasonable values. Users will get information on who wrote the configuration profile then when executing a nf-core pipeline and can report back if there are things missing for example. +Please also make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` and `params.config_profile_url` set to reasonable values. +Users will get information on who wrote the configuration profile then when executing a nf-core pipeline and can report back if there are things missing for example. ## Checking user hostnames If your cluster has a set of consistent hostnames, nf-core pipelines can check that users are using your profile. Add one or more hostname substrings to `params.hostnames` under a key that matches the profile name. -If the user's hostname contains this string at the start of a run or when a run fails and their profile -does not contain the profile name, a warning message will be printed. +If the user's hostname contains this string at the start of a run or when a run fails and their profile does not contain the profile name, a warning message will be printed. ### Testing @@ -80,7 +85,8 @@ nextflow run nf-core/rnaseq --reads '*_R{1,2}.fastq.gz' --genome GRCh37 -c '[pat ### Documentation -You will have to create a [Markdown document](https://www.markdownguide.org/getting-started/) outlining the details required to use the custom config file within your organisation. You might orientate yourself using the [Template](docs/template.md) that we provide and filling out the information for your cluster there. +You will have to create a [Markdown document](https://www.markdownguide.org/getting-started/) outlining the details required to use the custom config file within your organisation. +You might orientate yourself using the [Template](docs/template.md) that we provide and filling out the information for your cluster there. See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs) for examples. @@ -113,21 +119,35 @@ Currently documentation is available for the following systems: ### Uploading to `nf-core/configs` -[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. +[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. +Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. +You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. +You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. -## Adding a new pipeline specific config +## Adding a new pipeline-specific config -Follow the previous guidelines to create a nf-core/configs, update the pipeline specific documentation, and create the necessary files. -Replace `` with the pipeline name for all the instructions in this guidelines. -In a similar manner, replace `` by the name of the profile. +Here we will be adding an institutional pipeline-specific config. + +:warning: Replace `` with the pipeline name for all the instructions in this guidelines. +:warning: In a similar manner, replace `` by the name of the profile. + +Currently the `nextflow.config` file from any pipeline, load the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config), which load the institutional configuration file based on the profile ``. + +We're adding within the `nextflow.config` from the pipeline ``, the loading of the `pipeline/.config` from the [`nf-core/configs`](https://github.com/nf-core/configs) repo, which load the `` specific institution configuration file based on the profile ``. + +`nextflow.config` from pipeline will load first institutional configuration file and then the pipeline-specific institutional configuration file. +Each configuration file will add new params and overwrite the params already existing. + +Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` set to reasonable values. +Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. ### Pipeline specific documentation -Currently documentation is available for the following pipelines within the specific profile: +Currently documentation is available for the following pipeline within the specific profile: * sarek * [MUNIN](docs/pipeline/sarek/munin.md) @@ -136,7 +156,8 @@ Currently documentation is available for the following pipelines within the spec :warning: **This has to be done on a fork of the `nf-core/` repository.** -If not already present, add to `nextflow.config` +[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/` repository to your own GitHub account. +Within the local clone of your fork, if not already present, add to `nextflow.config` ```Groovy // Load nf-core/ custom profiles from different Institutions @@ -147,11 +168,16 @@ try { } ``` +Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/` GitHub repo with the appropriate information. + +We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. + ### Create the specific nf-core/configs for the pipeline :warning: This has to be done on a fork of the `nf-core/configs` repository. -If not already created, add the `pipeline/.config` file, and add the profile to the profile scope +[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. +Within the local clone of your fork, if not already created, create the `pipeline/.config` file, and add your custom profile to the profile scope ```Groovy profiles { @@ -159,10 +185,12 @@ profiles { } ``` -Add the `conf/pipeline//.config` file with all pipeline specific params, `withLabel` or any other configuration profile. +Add the custom configuration file to the `conf/pipeline//` directory. +Add the documentation file to the `docs/pipeline//` directory. +You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. -Add the `docs/pipeline//.md` file with documentation on what are the specificity of this configs. +Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. ## Help -If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). +If you have any questions or issues please send us a message on [Slack](https://nfcore.slack.com/channels/configs). From 45cec1ac2ff10fd7c6e3b910dd5cde21920182f3 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 10:10:05 +0100 Subject: [PATCH 14/32] Add specific sarek config base, munin specific sarek config and docs rename file improve configs remove unnecessary params --- conf/pipeline/sarek/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index b2670a9..5d4f490 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -21,4 +21,4 @@ process { container = '' } } -} \ No newline at end of file +} From 83f677216bd8e00a60146aec5a409263da7f0e81 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:13:13 +0100 Subject: [PATCH 15/32] Update pipeline/sarek.config --- pipeline/sarek.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/sarek.config b/pipeline/sarek.config index 9457972..aed6a1d 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -10,4 +10,4 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } -} \ No newline at end of file +} From 25d4e92935e73a621c4939baf86b9364cb4d950e Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 15:32:35 +0100 Subject: [PATCH 16/32] update docs --- docs/pipeline/sarek/munin.md | 4 ++++ pipeline/sarek.config | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index ce411c1..7589a9a 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -19,4 +19,8 @@ Specific configurations for MUNIN has been made for sarek. * Path to `vep_cache`: `/data1/cache/VEP/` * Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` * Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` +<<<<<<< HEAD * Load module `Sentieon` for Processes with `sentieon` labels +======= +* Load module `Sentieon` for Processes with `sentieon` labels +>>>>>>> 5dc5555... update docs diff --git a/pipeline/sarek.config b/pipeline/sarek.config index aed6a1d..9457972 100644 --- a/pipeline/sarek.config +++ b/pipeline/sarek.config @@ -10,4 +10,4 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" } -} +} \ No newline at end of file From 9d23f7a266af7e53eda969bb461e385659fbb0b4 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:53:49 +0100 Subject: [PATCH 17/32] Update conf/pipeline/sarek/munin.config --- conf/pipeline/sarek/munin.config | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 5d4f490..8c4e1a9 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -4,6 +4,14 @@ params { // Specific nf-core/configs params config_profile_contact = 'Maxime Garcia (@MaxUlysse)' config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' +<<<<<<< HEAD +======= + + // Specific nf-core/sarek configs params + max_memory = 752.GB + max_cpus = 46 + max_time = 72.h +>>>>>>> 946bf37... Update conf/pipeline/sarek/munin.config // Specific nf-core/sarek params annotation_cache = true From f6c2e4c65f6b1f355e0c248bd392e75f77c3c321 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 15:54:29 +0100 Subject: [PATCH 18/32] Update docs/pipeline/sarek/munin.md Update conf/pipeline/sarek/munin.config testing out module and containers collision --- conf/pipeline/sarek/munin.config | 8 -------- docs/pipeline/sarek/munin.md | 4 ---- 2 files changed, 12 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 8c4e1a9..5d4f490 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -4,14 +4,6 @@ params { // Specific nf-core/configs params config_profile_contact = 'Maxime Garcia (@MaxUlysse)' config_profile_description = 'nf-core/sarek MUNIN profile provided by nf-core/configs' -<<<<<<< HEAD -======= - - // Specific nf-core/sarek configs params - max_memory = 752.GB - max_cpus = 46 - max_time = 72.h ->>>>>>> 946bf37... Update conf/pipeline/sarek/munin.config // Specific nf-core/sarek params annotation_cache = true diff --git a/docs/pipeline/sarek/munin.md b/docs/pipeline/sarek/munin.md index 7589a9a..ce411c1 100644 --- a/docs/pipeline/sarek/munin.md +++ b/docs/pipeline/sarek/munin.md @@ -19,8 +19,4 @@ Specific configurations for MUNIN has been made for sarek. * Path to `vep_cache`: `/data1/cache/VEP/` * Path to `pon`: `/data1/PON/vcfs/BTB.PON.vcf.gz` * Path to `pon_index`: `/data1/PON/vcfs/BTB.PON.vcf.gz.tbi` -<<<<<<< HEAD * Load module `Sentieon` for Processes with `sentieon` labels -======= -* Load module `Sentieon` for Processes with `sentieon` labels ->>>>>>> 5dc5555... update docs From 880551d246badf5e93c1ff6a68fcd28541c9e0e5 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 20 Nov 2019 16:00:05 +0100 Subject: [PATCH 19/32] Apply suggestions from code review --- conf/pipeline/sarek/munin.config | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 5d4f490..1327936 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -13,6 +13,7 @@ params { vep_cache = '/data1/cache/VEP/' } +<<<<<<< HEAD // Specific nf-core/sarek process configuration process { withLabel:sentieon { @@ -21,4 +22,10 @@ process { container = '' } } +======= +// Specific nf-core/sarek configuration +withLabel:sentieon { + module = {params.sentieon ? 'sentieon/201808.05' : ''} + if (params.sentieon) container = '' +>>>>>>> bfcbcea... Apply suggestions from code review } From ebdfc2a76842ab5cc18348a9f03b6de1cbe6d5b7 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 20 Nov 2019 16:03:27 +0100 Subject: [PATCH 20/32] fix container/module collision --- conf/pipeline/sarek/munin.config | 7 ------- 1 file changed, 7 deletions(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 1327936..5d4f490 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -13,7 +13,6 @@ params { vep_cache = '/data1/cache/VEP/' } -<<<<<<< HEAD // Specific nf-core/sarek process configuration process { withLabel:sentieon { @@ -22,10 +21,4 @@ process { container = '' } } -======= -// Specific nf-core/sarek configuration -withLabel:sentieon { - module = {params.sentieon ? 'sentieon/201808.05' : ''} - if (params.sentieon) container = '' ->>>>>>> bfcbcea... Apply suggestions from code review } From d1c4ba89bd34f12a758d2780d4e9d7ef93372432 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 21 Nov 2019 14:21:32 +0100 Subject: [PATCH 21/32] actually belong to nf-core/configs/munin From 6c516282bc0237bf354b94f673ea1eb1c78623e7 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Thu, 21 Nov 2019 15:25:12 +0100 Subject: [PATCH 22/32] Update README.md From 52cacd6c97015fb885048d79bc175f0d21a0824c Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 12:00:40 +0100 Subject: [PATCH 23/32] Update README.md Update README.md Co-Authored-By: James A. Fellows Yates # This is a combination of 2 commits. Update README.md Update README.md Co-Authored-By: James A. Fellows Yates From 89de9a799ea40ce44f13bfbc81600723158748f6 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Fri, 22 Nov 2019 14:42:40 +0100 Subject: [PATCH 24/32] improve syntax From 89e6d8cd275ea9baa4862cc58653b77afed62010 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Fri, 22 Nov 2019 13:55:08 +0100 Subject: [PATCH 25/32] Update conf/munin.config From f9fe789ef43a493958ab3fe7de2e2e891e57c0e8 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 10:30:24 +0100 Subject: [PATCH 26/32] update README From f4115bc1a92d9c14023c1365157da19668d45bb1 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 26 Nov 2019 14:10:05 +0100 Subject: [PATCH 27/32] Update README.md Co-Authored-By: Phil Ewels --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f42311..fe917e3 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,9 @@ We will be notified automatically when you have created your pull request, and p ## Adding a new pipeline-specific config -Here we will be adding an institutional pipeline-specific config. +Sometimes it may be desirable to have configuration options for an institute that are specific to a single nf-core pipeline. +Such options should not be added to the main institutional config, as this will be applied to all pipelines. +Instead, we can create a pipeline-specific institutional config file. :warning: Replace `` with the pipeline name for all the instructions in this guidelines. :warning: In a similar manner, replace `` by the name of the profile. From 68c709432fa239fa35bbcb37c257b3d6c50dffb8 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 26 Nov 2019 14:17:32 +0100 Subject: [PATCH 28/32] Apply suggestions from code review Co-Authored-By: Phil Ewels --- README.md | 18 ++++++++++-------- conf/munin.config | 1 - 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index fe917e3..d2ae637 100644 --- a/README.md +++ b/README.md @@ -134,17 +134,18 @@ Sometimes it may be desirable to have configuration options for an institute tha Such options should not be added to the main institutional config, as this will be applied to all pipelines. Instead, we can create a pipeline-specific institutional config file. -:warning: Replace `` with the pipeline name for all the instructions in this guidelines. -:warning: In a similar manner, replace `` by the name of the profile. +:warning: Remember to replace the `` and `` placeholders with the pipeline name and profile name in the following examples -Currently the `nextflow.config` file from any pipeline, load the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config), which load the institutional configuration file based on the profile ``. +Institutional configs work because the pipeline `nextflow.config` file loads the [`nf-core/configs/nfcore_custom.config` config file](https://github.com/nf-core/configs/blob/master/nfcore_custom.config), which in turn loads the institutional configuration file based on the profile `` supplied on the command line. -We're adding within the `nextflow.config` from the pipeline ``, the loading of the `pipeline/.config` from the [`nf-core/configs`](https://github.com/nf-core/configs) repo, which load the `` specific institution configuration file based on the profile ``. +To add in pipeline-specific institutional configs, we add a second `includeConfig` call in the pipeline `nextflow.config` file, which loads the `pipeline/.config` file from the [`nf-core/configs`](https://github.com/nf-core/configs) repo. This file has `` specific institution configuration again with different profiles ``. -`nextflow.config` from pipeline will load first institutional configuration file and then the pipeline-specific institutional configuration file. +The pipeline `nextflow.config` file should first load the generic institutional configuration file and then the pipeline-specific institutional configuration file. Each configuration file will add new params and overwrite the params already existing. -Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` set to reasonable values. +Note that pipeline-specific configs are not required and should only be added if needed. + +Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` to the top of `pipeline/.config` and set to reasonable values. Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. ### Pipeline specific documentation @@ -159,9 +160,9 @@ Currently documentation is available for the following pipeline within the speci :warning: **This has to be done on a fork of the `nf-core/` repository.** [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/` repository to your own GitHub account. -Within the local clone of your fork, if not already present, add to `nextflow.config` +Within the local clone of your fork, if not already present, add the following to `nextflow.config` **after** the code that loads the generic nf-core/configs config file: -```Groovy +```nextflow // Load nf-core/ custom profiles from different Institutions try { includeConfig "${params.custom_config_base}/pipeline/.config" @@ -192,6 +193,7 @@ Add the documentation file to the `docs/pipeline//` directory. You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. +In the pull-request description, add a link to the repository specific pull-request(s) that use this new code. Both PRs will need to be merged at the approximately the same time. ## Help diff --git a/conf/munin.config b/conf/munin.config index df45787..09df9c8 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -20,7 +20,6 @@ process { maxForks = 46 } -// To use singularity, use nextflow run -profile munin singularity { enabled = true cacheDir = '/data1/containers/' From 25f3bd68f878276d1ed97f0fb2f5e082b2df35d0 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 15:21:53 +0100 Subject: [PATCH 29/32] container = null --- conf/pipeline/sarek/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pipeline/sarek/munin.config b/conf/pipeline/sarek/munin.config index 5d4f490..5265007 100644 --- a/conf/pipeline/sarek/munin.config +++ b/conf/pipeline/sarek/munin.config @@ -18,7 +18,7 @@ process { withLabel:sentieon { if (params.sentieon) { module = 'sentieon/201808.05' - container = '' + container = null } } } From 8a73e7f2f826738b7e8b830dd55497e5eb1d98c3 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 27 Nov 2019 13:16:08 +0100 Subject: [PATCH 30/32] better explanation --- README.md | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d2ae637..934ef12 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) -A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. +A repository for hosting Nextflow configuration files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents @@ -80,7 +80,7 @@ If you want to add a new custom config file to `nf-core/configs` please test tha ```bash ## Example command for nf-core/rnaseq -nextflow run nf-core/rnaseq --reads '*_R{1,2}.fastq.gz' --genome GRCh37 -c '[path to custom config]' +nextflow run nf-core/rnaseq --reads '*_R{1,2}.fastq.gz' --genome GRCh37 -c '/path/to/custom.config' ``` ### Documentation @@ -119,7 +119,7 @@ Currently documentation is available for the following systems: ### Uploading to `nf-core/configs` -[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. +[Fork](https://help.github.com/articles/fork-a-repo/) the [`nf-core/configs`](https://github.com/nf-core/configs/) repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. @@ -130,7 +130,7 @@ We will be notified automatically when you have created your pull request, and p ## Adding a new pipeline-specific config -Sometimes it may be desirable to have configuration options for an institute that are specific to a single nf-core pipeline. +Sometimes it may be desirable to have configuration options for an institute that are specific to a single nf-core pipeline. Such options should not be added to the main institutional config, as this will be applied to all pipelines. Instead, we can create a pipeline-specific institutional config file. @@ -138,26 +138,24 @@ Instead, we can create a pipeline-specific institutional config file. Institutional configs work because the pipeline `nextflow.config` file loads the [`nf-core/configs/nfcore_custom.config` config file](https://github.com/nf-core/configs/blob/master/nfcore_custom.config), which in turn loads the institutional configuration file based on the profile `` supplied on the command line. -To add in pipeline-specific institutional configs, we add a second `includeConfig` call in the pipeline `nextflow.config` file, which loads the `pipeline/.config` file from the [`nf-core/configs`](https://github.com/nf-core/configs) repo. This file has `` specific institution configuration again with different profiles ``. +To add in pipeline-specific institutional configs, we add a second `includeConfig` call in the pipeline `nextflow.config` file, which loads the `pipeline/.config` file from the [`nf-core/configs`](https://github.com/nf-core/configs) repo. +This file has `` specific institution configuration again with different profiles ``. The pipeline `nextflow.config` file should first load the generic institutional configuration file and then the pipeline-specific institutional configuration file. Each configuration file will add new params and overwrite the params already existing. Note that pipeline-specific configs are not required and should only be added if needed. -Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` to the top of `pipeline/.config` and set to reasonable values. -Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. - -### Pipeline specific documentation +### Pipeline-specific documentation Currently documentation is available for the following pipeline within the specific profile: * sarek * [MUNIN](docs/pipeline/sarek/munin.md) -### Enabling the specific configs within a specific pipeline +### Enabling pipeline-specific configs within a pipeline -:warning: **This has to be done on a fork of the `nf-core/` repository.** +:warning: **This has to be done on a fork of the `nf-core/` repository.** [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/` repository to your own GitHub account. Within the local clone of your fork, if not already present, add the following to `nextflow.config` **after** the code that loads the generic nf-core/configs config file: @@ -175,12 +173,16 @@ Commit and push these changes to your local clone on GitHub, and then [create a We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. -### Create the specific nf-core/configs for the pipeline +### Create the pipeline-specific `nf-core/configs` files -:warning: This has to be done on a fork of the `nf-core/configs` repository. +:warning: This has to be done on a fork of the [`nf-core/configs`](https://github.com/nf-core/configs/) repository. -[Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. -Within the local clone of your fork, if not already created, create the `pipeline/.config` file, and add your custom profile to the profile scope +[Fork](https://help.github.com/articles/fork-a-repo/) the [`nf-core/configs`](https://github.com/nf-core/configs/) repository to your own GitHub account. +And add or edit the following files in the local clone of your fork. + +* `pipeline/.config` + +If not already created, create the `pipeline/.config` file, and add your custom profile to the profile scope ```Groovy profiles { @@ -188,12 +190,24 @@ profiles { } ``` +* `conf/pipeline//.config` + Add the custom configuration file to the `conf/pipeline//` directory. +Make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` to the top of `pipeline/.config` and set to reasonable values. +Users will get information on who wrote the pipeline-specific configuration profile then when executing the nf-core pipeline and can report back if there are things missing for example. + +* `docs/pipeline//.md` + Add the documentation file to the `docs/pipeline//` directory. You will also need to edit and add your custom profile to the [`README.md`](https://github.com/nf-core/configs/blob/master/README.md) file in the top-level directory of the clone. +* `README.md` + +Edit this file, and add the new pipeline-specific institutional profile to the list in the section Pipeline specific documentation + Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. -In the pull-request description, add a link to the repository specific pull-request(s) that use this new code. Both PRs will need to be merged at the approximately the same time. +In the pull-request description, add a link to the repository specific pull-request(s) that use this new code. +Both PRs will need to be merged at the approximately the same time. ## Help From 7f61d2f81bac44a430fa23c9a00eb75d11a2f118 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 27 Nov 2019 13:18:39 +0100 Subject: [PATCH 31/32] fix TOC --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 934ef12..99e73db 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ A repository for hosting Nextflow configuration files containing custom paramete * [Testing](#testing) * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Adding a new pipeline specific config](#adding-a-new-pipeline-specific-config) - * [Pipeline Specific Documentation](#pipeline-specific-documentation) - * [Enabling the specific configs within a specific pipeline](#enabling-the-specific-configs-within-a-specific-pipeline) - * [Create the specific `nf-core/configs` for the pipeline](#create-the-specific-nf-coreconfigs-for-the-pipeline) +* [Adding a new pipeline-specific config](#adding-a-new-pipeline-specific-config) + * [Pipeline-specific documentation](#pipeline-specific-documentation) + * [Enabling pipeline-specific configs within a pipeline](#enabling-pipeline-specific-configs-within-a-pipeline) + * [Create the pipeline-specific `nf-core/configs` files](#create-the-pipeline-specific-nf-coreconfigs-files) * [Help](#help) ## Using an existing config From 752fca053085e78fa1e87c0cea604ee4a0831e92 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 27 Nov 2019 13:25:52 +0100 Subject: [PATCH 32/32] fix level of hostname --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99e73db..7b3bbb2 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ See [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf) for Please also make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` and `params.config_profile_url` set to reasonable values. Users will get information on who wrote the configuration profile then when executing a nf-core pipeline and can report back if there are things missing for example. -## Checking user hostnames +### Checking user hostnames If your cluster has a set of consistent hostnames, nf-core pipelines can check that users are using your profile. Add one or more hostname substrings to `params.hostnames` under a key that matches the profile name.