From 13df59bfaf45d385fc7d745b31b9a174731da0ef Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 12 Mar 2019 17:57:31 +0000 Subject: [PATCH 001/138] remove singularity support reasons: * the kernel running on the compute nodes does not support overlayfs und thus containers downloaded from public registries won't run because they lack the /lustre mountpoint * building containers manually using the Singularity file would work, however nf-core pipelines will stop providing these files in future releases --- conf/mendel.config | 2 +- docs/mendel.md | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/conf/mendel.config b/conf/mendel.config index 940a914..0715b42 100644 --- a/conf/mendel.config +++ b/conf/mendel.config @@ -11,7 +11,7 @@ manifest { } process { - beforeScript = {'module load Singularity; module load Miniconda3'} + beforeScript = 'module load Miniconda3/4.6.7' executor = 'pbspro' clusterOptions = { "-P $params.project" } } diff --git a/docs/mendel.md b/docs/mendel.md index 3f3db20..ae0ad4c 100644 --- a/docs/mendel.md +++ b/docs/mendel.md @@ -2,27 +2,15 @@ All nf-core pipelines have been successfully configured for use on the MENDEL CLUSTER at the Gregor Mendel Institute (GMI). -To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be downloaded from ['bioconda'](https://bioconda.github.io/). +To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be resolved via [bioconda](https://bioconda.github.io/). -Theoretically, using `-profile singularity,mendel` would download a docker image containing all of the required software, and convert it to a Singularity image before execution of the pipeline. However, there is a regression in the Singularity deployment on MENDEL which renders containers downloaded from public repositories unusable because they lack the /lustre mountpoint. - -If you want to run the pipeline containerized anyway you will have to build the image yourself (on a machine where you have root access) using the provided `Singularity` file in the pipeline repository: - -```bash -cd /path/to/pipeline-repository -echo 'mkdir /lustre > Singularity' -singularity build nf-core-methylseq-custom.simg Singularity -``` - -After you copied the container image to the cluster filesystem, make sure to pass the path to the image to the pipeline with `-with-singularity /path/to/nf-core-methylseq-custom.simg` - -Before running the pipeline you will need to load Nextflow and Conda using the environment module system on MENDEL. You can do this by issuing the commands below: +Before running the pipeline you will need to load Conda using the environment module system on MENDEL. You can do this by issuing the commands below: ```bash ## Load Nextflow and Conda environment modules module purge module load Nextflow -module load Miniconda3 # not needed if using Singularity +module load Miniconda/4.6.7 ``` >NB: You will need an account to use the HPC cluster in order to run the pipeline. If in doubt contact the HPC team. From aaa7be049a9589e79cce863fd6d582fb8fcf7799 Mon Sep 17 00:00:00 2001 From: KochTobi Date: Sat, 16 Mar 2019 14:11:59 +0100 Subject: [PATCH 002/138] Update cfc config * Change singularity version documentation --- docs/cfc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cfc.md b/docs/cfc.md index e1cccac..5cf87d1 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load devel/java_jdk/1.8.0u121 -module load qbic/singularity_slurm/3.0.1 +module load qbic/singularity_slurm/3.0.3 ``` From 0929349303821693e66c3fc9f9929b4a6bbc00c9 Mon Sep 17 00:00:00 2001 From: maxibor Date: Mon, 18 Mar 2019 11:13:11 +0100 Subject: [PATCH 003/138] add queueSize limited to 5 --- conf/shh.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/shh.config b/conf/shh.config index 43d192f..92db897 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -13,6 +13,7 @@ singularity { process { executor = 'slurm' queue = 'short' + queueSize = 5 } params { From 76a5fcf0776e3391aeeda4d3e9d1a9f193e1dde7 Mon Sep 17 00:00:00 2001 From: maxibor Date: Wed, 20 Mar 2019 13:44:13 +0100 Subject: [PATCH 004/138] switch to queueSize 16 for shh config --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 92db897..159de89 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -13,7 +13,7 @@ singularity { process { executor = 'slurm' queue = 'short' - queueSize = 5 + queueSize = 16 } params { From 51c2a54399e0a743eccd38ca2536674e7ee979f2 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Wed, 17 Apr 2019 16:59:38 -0400 Subject: [PATCH 005/138] added nyu prince and bigpurple configs --- conf/bigpurple.config | 23 +++++++++++++++++++++++ conf/prince.config | 21 +++++++++++++++++++++ nfcore_custom.config | 2 ++ 3 files changed, 46 insertions(+) create mode 100644 conf/bigpurple.config create mode 100644 conf/prince.config diff --git a/conf/bigpurple.config b/conf/bigpurple.config new file mode 100644 index 0000000..b7850db --- /dev/null +++ b/conf/bigpurple.config @@ -0,0 +1,23 @@ +params { + config_profile_description = """ + NYU School of Medicine BigPurple cluster profile provided by nf-core/configs. + module load both singularity/3.1 and squashfs-tools/4.3 before running the pipeline with this profile! + Also consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity + """.stripIndents() + config_profile_contact = 'Tobias Schraink (@tobsecret)' + config_profile_url = 'https://genome.med.nyu.edu/hpcf/wiki/Manual:Cluster_User_Guide' +} + +singularity { + enabled = true + autoMounts = true +} + +process { + beforeScript = """ + module load singularity/3.1 + module load squashfs-tools/4.3 + """ + .stripIndent() + executor = 'slurm' +} diff --git a/conf/prince.config b/conf/prince.config new file mode 100644 index 0000000..5fc07c6 --- /dev/null +++ b/conf/prince.config @@ -0,0 +1,21 @@ +params { + config_profile_description = """ + NYU prince cluster profile provided by nf-core/configs. + Please consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity + """.stripIndent() + config_profile_contact = 'Tobias Schraink (@tobsecret)' + config_profile_url = 'https://wikis.nyu.edu/display/NYUHPC/Clusters+-+Prince' +} + +singularity { + enabled = true +} + +process { + beforeScript = """ + module load singularity/3.1.0 + module load squashfs/4.3 + """ + .stripIndent() + executor = 'slurm' +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 61f43a9..7374fa0 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -26,6 +26,8 @@ profiles { uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } + bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } } // If user hostnames contain one of these substring and they are From 4a915e330a7e0bb8e02c4e5d7739f4364b21ae32 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Wed, 17 Apr 2019 17:15:16 -0400 Subject: [PATCH 006/138] added warnings to run on SCRATCH --- conf/bigpurple.config | 3 ++- conf/prince.config | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conf/bigpurple.config b/conf/bigpurple.config index b7850db..967cc5f 100644 --- a/conf/bigpurple.config +++ b/conf/bigpurple.config @@ -1,7 +1,8 @@ params { config_profile_description = """ NYU School of Medicine BigPurple cluster profile provided by nf-core/configs. - module load both singularity/3.1 and squashfs-tools/4.3 before running the pipeline with this profile! + module load both singularity/3.1 and squashfs-tools/4.3 before running the pipeline with this profile!! + Run from your scratch or lab directory - Nextflow makes a lot of files!! Also consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity """.stripIndents() config_profile_contact = 'Tobias Schraink (@tobsecret)' diff --git a/conf/prince.config b/conf/prince.config index 5fc07c6..76b32ed 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -1,6 +1,7 @@ params { config_profile_description = """ NYU prince cluster profile provided by nf-core/configs. + Run from your scratch directory, the output files may be large! Please consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity """.stripIndent() config_profile_contact = 'Tobias Schraink (@tobsecret)' From 1772f4119c391a4df4d6f7d4219b537d375485ec Mon Sep 17 00:00:00 2001 From: maxibor Date: Thu, 18 Apr 2019 11:19:53 +0200 Subject: [PATCH 007/138] adding igenomes to shh config --- conf/shh.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 159de89..020be37 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -2,7 +2,7 @@ params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_url = 'https://shh.mpg.de' + config_profile_url = 'https: //shh.mpg.de' } singularity { @@ -20,4 +20,6 @@ params { max_memory = 256.GB max_cpus = 32 max_time = 2.h -} + //Illumina iGenomes reference file path + igenomes_base = "/projects1/public_data/igenomes/" +} \ No newline at end of file From 9d9ac4c3bb33f41561c402b122c36fbed8771e28 Mon Sep 17 00:00:00 2001 From: maxibor Date: Thu, 18 Apr 2019 11:22:54 +0200 Subject: [PATCH 008/138] remove whitespace --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 020be37..d2feaff 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -2,7 +2,7 @@ params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_url = 'https: //shh.mpg.de' + config_profile_url = 'https://shh.mpg.de' } singularity { From 4ba831705dc9bf0fb14764ffedf99f97cc1e9c75 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Thu, 18 Apr 2019 14:13:02 -0400 Subject: [PATCH 009/138] added nyu prince and bigpurple md files, added singularity cacheDir to prince and bigpurple configs --- conf/bigpurple.config | 9 ++++++--- conf/prince.config | 7 +++++-- docs/bigpurple.md | 23 +++++++++++++++++++++++ docs/prince.md | 19 +++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 docs/bigpurple.md create mode 100644 docs/prince.md diff --git a/conf/bigpurple.config b/conf/bigpurple.config index 967cc5f..5fdf7e8 100644 --- a/conf/bigpurple.config +++ b/conf/bigpurple.config @@ -1,17 +1,20 @@ +singularityDir = "/gpfs/scratch/${USER}/singularity_images_nextflow" + params { config_profile_description = """ NYU School of Medicine BigPurple cluster profile provided by nf-core/configs. module load both singularity/3.1 and squashfs-tools/4.3 before running the pipeline with this profile!! Run from your scratch or lab directory - Nextflow makes a lot of files!! - Also consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity - """.stripIndents() + Also consider running the pipeline on a compute node (srun --pty /bin/bash -t=01:00:00) the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node and will take some time. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in $singularityDir + """.stripIndent() config_profile_contact = 'Tobias Schraink (@tobsecret)' - config_profile_url = 'https://genome.med.nyu.edu/hpcf/wiki/Manual:Cluster_User_Guide' + config_profile_url = 'https://github.com/nf-core/configs/blob/master/docs/bigpurple.md' } singularity { enabled = true autoMounts = true + cacheDir = singularityDir } process { diff --git a/conf/prince.config b/conf/prince.config index 76b32ed..6ae4ee8 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -1,15 +1,18 @@ +singularityDir = "$SCRATCH/singularity_images_nextflow" + params { config_profile_description = """ NYU prince cluster profile provided by nf-core/configs. Run from your scratch directory, the output files may be large! - Please consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in ./work/singularity + Please consider running the pipeline on a compute node the first time, as it will be pulling the docker image, which will be converted into a singularity image, which is heavy on the login node. Subsequent runs can be done on the login node, as the docker image will only be pulled and converted once. By default the images will be stored in $singularityDir """.stripIndent() config_profile_contact = 'Tobias Schraink (@tobsecret)' - config_profile_url = 'https://wikis.nyu.edu/display/NYUHPC/Clusters+-+Prince' + config_profile_url = 'https://github.com/nf-core/configs/blob/master/docs/prince.md' } singularity { enabled = true + cacheDir = singularityDir } process { diff --git a/docs/bigpurple.md b/docs/bigpurple.md new file mode 100644 index 0000000..e3b8dfb --- /dev/null +++ b/docs/bigpurple.md @@ -0,0 +1,23 @@ +# nf-core/configs: BigPurple Configuration +## nf-core pipelines that use this repo +All nf-core pipelines that use this config repo (which is most), can be run on BigPurple. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. Once in the interactive session: + +``` +module load singularity/3.1 +module load squashfs-tools/4.3 +``` + +Now, run the pipeline of your choice with `-profile bigpurple`. This will download and launch the bigpurple.config which has been pre-configured with a setup suitable for the BigPurple cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. +An example commandline: + +`nextflow run nf-core/ -profile bigpurple ` + +## nf-core pipelines that do not use this repo +If the pipeline has not yet been configured to use this config, then you will have to do it manually. +git clone this repo, copy the `bigpurple.config` from the conf folder and then you can invoke the pipeline like this: + +`nextflow run nf-core/ -c bigpurple.config ` + + +>NB: You will need an account to use the HPC cluster BigPurple in order to run the pipeline. If in doubt contact MCIT. +>NB: You will need to install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason there is no module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update yourself: `nextflow self-update` diff --git a/docs/prince.md b/docs/prince.md new file mode 100644 index 0000000..9a49ee0 --- /dev/null +++ b/docs/prince.md @@ -0,0 +1,19 @@ +# nf-core/configs: Prince Configuration +## nf-core pipelines that use this repo +All nf-core pipelines that use this config repo (which is most), can be run on prince. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. + +Now, run the pipeline of your choice with `-profile prince`. This will download and launch the prince.config which has been pre-configured with a setup suitable for the prince cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. This step **takes time**!! +An example commandline: + +`nextflow run nf-core/ -profile prince ` + +## nf-core pipelines that do not use this repo +If the pipeline has not yet been configured to use this config, then you will have to do it manually. +git clone this repo, copy the `prince.config` from the conf folder and then you can invoke the pipeline like this: + +`nextflow run nf-core/ -c prince.config ` + + +>NB: You will need an account to use the HPC cluster Prince in order to run the pipeline. If in doubt contact the HPC admins. +>NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update` + From b6c1d3ba518ce5c046f698bddd644973448381df Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Thu, 18 Apr 2019 14:23:17 -0400 Subject: [PATCH 010/138] added bigpurple and prince doc links in README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index acb2ea4..eadd3bb 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) +* [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) @@ -93,6 +94,7 @@ Currently documentation is available for the following clusters: * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) * [PHOENIX](docs/phoenix.md) +* [PRINCE](docs/prince.md) * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) * [UPPMAX-DEVEL](docs/uppmax-devel.md) From d91b06d884dfbb86f537662460f07e9f2859dbd6 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Thu, 18 Apr 2019 14:25:27 -0400 Subject: [PATCH 011/138] added missing line in prince and bigpurple docs --- docs/bigpurple.md | 1 + docs/prince.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/bigpurple.md b/docs/bigpurple.md index e3b8dfb..19b0352 100644 --- a/docs/bigpurple.md +++ b/docs/bigpurple.md @@ -20,4 +20,5 @@ git clone this repo, copy the `bigpurple.config` from the conf folder and then y >NB: You will need an account to use the HPC cluster BigPurple in order to run the pipeline. If in doubt contact MCIT. + >NB: You will need to install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason there is no module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update yourself: `nextflow self-update` diff --git a/docs/prince.md b/docs/prince.md index 9a49ee0..12f5e65 100644 --- a/docs/prince.md +++ b/docs/prince.md @@ -15,5 +15,6 @@ git clone this repo, copy the `prince.config` from the conf folder and then you >NB: You will need an account to use the HPC cluster Prince in order to run the pipeline. If in doubt contact the HPC admins. + >NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update` From efd831a8a34fe7cae2225faf71eaa72383c13edb Mon Sep 17 00:00:00 2001 From: maxibor Date: Thu, 25 Apr 2019 15:48:20 +0200 Subject: [PATCH 012/138] add automounts for singularity --- conf/shh.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index d2feaff..fd16e9e 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -2,11 +2,12 @@ params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_url = 'https://shh.mpg.de' + config_profile_url = 'https: //shh.mpg.de' } singularity { enabled = true + autoMounts = true cacheDir = "/projects1/singularity_scratch/cache/" } From de1b2aa7d27bff386595f802bab47939690ef022 Mon Sep 17 00:00:00 2001 From: maxibor Date: Thu, 25 Apr 2019 15:50:48 +0200 Subject: [PATCH 013/138] remove autoformat added space --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index fd16e9e..2b195a6 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -2,7 +2,7 @@ params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133)' - config_profile_url = 'https: //shh.mpg.de' + config_profile_url = 'https://shh.mpg.de' } singularity { From 0acdcc381387338ce35bbf42a89f44cac3cb6f92 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sat, 4 May 2019 23:20:15 +0200 Subject: [PATCH 014/138] Add Aquila Config Singapore --- README.md | 1 + conf/aquila.config | 26 ++++++++++++++++++++++++++ docs/aquila.md | 8 ++++++++ nfcore_custom.config | 1 + 4 files changed, 36 insertions(+) create mode 100644 conf/aquila.config create mode 100644 docs/aquila.md diff --git a/README.md b/README.md index eadd3bb..d16b13e 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: +* [AQUILA](docs/aquila.md) * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) diff --git a/conf/aquila.config b/conf/aquila.config new file mode 100644 index 0000000..05035d4 --- /dev/null +++ b/conf/aquila.config @@ -0,0 +1,26 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'A*STAR Aquila cluster profile provided by nf-core/configs.' + config_profile_contact = 'Andreas Wilm (@andreas-wilm)' + config_profile_url = 'https://www.a-star.edu.sg/' +} + +singularity { + enabled = true +} + +process { + beforeScript = { 'source /mnt/projects/rpd/rc/init.2017-04; module load miniconda3; set +u; source activate ' + params.container.replaceAll(/[\/:]/, '-').replaceAll(/nfcore/, 'nf-core') + "; set -u;" } + executor = 'sge' + clusterOptions = { "-l mem_free=" + task.memory.toString().replaceAll(/[\sB]/,'') } + penv = 'OpenMP' +} + +params { + igenomes_base = '/nfsmounts/igenomes' + max_memory = 128.GB + max_cpus = 28 + max_time = 48.h +} + + diff --git a/docs/aquila.md b/docs/aquila.md new file mode 100644 index 0000000..3e0575e --- /dev/null +++ b/docs/aquila.md @@ -0,0 +1,8 @@ +# nf-core/configs: Aquila Configuration + +All nf-core pipelines have been successfully configured for use on the Aquila cluster at A*STAR Singapore. + +To use, run the pipeline with `-profile aquila`. This will download and launch the [`aquila.config`](../conf/aquila.config) which has been pre-configured with a setup suitable for the Aquila cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +>NB: You will need an account to use the HPC cluster Aquila in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 7374fa0..17f51c5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From 441ddbcdab01871842451bf4b33d757d8c63d87a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 28 May 2019 15:25:10 +0200 Subject: [PATCH 015/138] Newest Update is 3.0.3 --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 5df94bf..0e69d4a 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -10,7 +10,7 @@ singularity { } process { - beforeScript = 'module load devel/singularity/3.0.1' + beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' } From 66eeeebe81d1aef1d8083e492484e47975d8760b Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 30 May 2019 11:15:38 -0700 Subject: [PATCH 016/138] Initial commit of CZBiohub --- conf/czbiohub.config | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 conf/czbiohub.config diff --git a/conf/czbiohub.config b/conf/czbiohub.config new file mode 100644 index 0000000..ff1523e --- /dev/null +++ b/conf/czbiohub.config @@ -0,0 +1,98 @@ +/* + * ------------------------------------------------- + * Nextflow config file for UPPMAX (milou / irma) + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + +docker { + enabled = true +} + +process { + executor = 'awsbatch' +} + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + workDir = "s3://czb-nextflow/rnaseq/" + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" + gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" + gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + } + } +} From ac804859f42a2bfa20a7f9c6fbbc27915729ed47 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Fri, 31 May 2019 00:47:06 +0100 Subject: [PATCH 017/138] Add nf-core logo --- README.md | 4 +--- docs/images/nfcore-configs_logo.png | Bin 0 -> 14072 bytes 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100755 docs/images/nfcore-configs_logo.png diff --git a/README.md b/README.md index 6a53642..75d6199 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ - - -# [nf-core/configs](https://github.com/nf-core/configs) +# ![nf-core/configs](docs/images/nfcore-configs_logo.png) [![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) diff --git a/docs/images/nfcore-configs_logo.png b/docs/images/nfcore-configs_logo.png new file mode 100755 index 0000000000000000000000000000000000000000..eef073868983145737377f81cb92f40132f3c866 GIT binary patch literal 14072 zcmaKTWmp_d(>3ny?gY2s9(?ft2`(Gl-QC@mU_leIgy8PBIKeHrySwYVxu5U%x7Vys zb$9htcTJyjI_9IA0wx+68Vn2!rXoOA69xu$3i^2v1qu3Us1FQ+z9G0tD}F+OKKxP4 zqoLnXodJ4oFfd3}|DLdJf}0!AA4%QibltU_EZx1nxLUw?d3kZ#ING_HeQ~znbaJ)I zJ{2Q_fuVs>l$HAAopX}wv^Ebr+^Rq}@wV^zLb^ES_C?aY3|Jw06oT z--@bqr2Vd}UUVFlVkb-H4%8+Vk6r;$v?0k`6_HV$CFoG zgFsFrWq6~;-vI1U2nC~pg2?_QPKq#k?+Rzt`OFtxHzv#`B{)Rd^N*%Wc{lU4xY^sMclgQ7~(ocU~sUpEhzIz_|mh^Ri zwT||GjZd-MuKGgB{>lzyg4yCdT=>RI%{hDTD_;Wb%T$4WW9iu#IIY+NS*f3%P+yfm z2Ep)f%U<-(P9Y94X&`!qUnez;Yh!_S0b3?o_T)1lR;U?KD-1DwF~yw&EL@RuD*JFc zW#3-uB(aJ2T^MPffK=muCB-=0U39A55k_zpl7<+%S zjXT+V2-LAt(ZSXvNw;8Ep87UZMBUN46%|i?Mu7N>Id%%+GXxR#ti3Qfk^uoe=i^%f z|0}Nx=??_pJh;+pUbDd>m50LV3xj9&^Uqyf_*=rQZ+2Gs?=K!g$%~nl?DjO*lVDH% zVDC+#&gAw)VlPbBt}ZP&=}6qIj^qjbstT9V0SnK2JT?Uy8|}t&*b8G@J2@DSiiQbC z#`_R_R48Fq(&fbTUZX~{m6JX{$Fey;!7z0$_?}p&C%-IEVee2mLMZUHnarBFQ9QPw z&LZW=^jn1g>f{4%^KMPo8?VUddKAPKFvL3HvhVCvLSmr}VDJriEll9>)~2+9m(Z(S zM+RagE@44SLvp9_9vU8v;A2pXrx1RR>F#*`Gq%ZILm|uJ{MtplYB#p&31HiH3Aj+>X`kKQEMxd;%AqIi2DNf>8U^dr z2ywG7ROi`nOPVC?mgh4(e)?we`XgN=LF#3svcc;mHRH2f$LB)@5rotgCqS<+5o<-q zC=UsG&7Sw1agEo?dUWSxe5vp0`#bA5Ff1Y4 z)vaL7TRiSHfsSVBe`DT96Vw00scw1#WDGJCYCu36_RFv-n?D}jkIXcPxovTz=wvVO zR;{(K)N^2JygQPg9*Lvm zqT`&wiqN6qAOeT}hqBx*IYFAsnAV#X?(ThFO*_dV+TvUjK*|tn4TE zB}xAz%da=j_`7+od!B@rd@ph&>+`31fO2S9;&RgH4h?Ii2IH?>#Z=Z;=84>XF*_b- z(d;NPNk43Med{tQIr;dPCyUCV@M5$FzR0*~rJ!_FbgJX4S|vh@Fn`02ldN4r0W%uX zy)VQMpW1KIYw944TrgcSbYoknf85==(I9-q&=dN-?U zJ*r|JXhw0S;AVgXxCIZO*(+Nwrg?S1laMx2;?|%Wum&xiBRi~uel>pFnVG1HxY#l) z4k%()Dt(vF$`uocgqh{Lp{MJow(KdJ)c)OfaX#R%lP!~`Cwi$rt^8o)?pCaozTb05 zce+b=2!_3^r-sww*mwNiZjp=Q?K&aODXH){$t1m4<<_A3uNLezyeHgZ&uq-7Jdm=A zN;h;yR|Ty-WFvZ1lvcQh5)@-2+Iy?dK>39SkMF&xpM`1FTv{H$BcWpOLP2Ny#^J*& zXLbnU;E}!-L-tUxc!*JL!;S11-Y{;FCVZLvDwkWxU5!9GnhXwZq+uEStD&w|H;?&Z zX5u#t!Mdqr?Q!#wq>xFR5c(6zn`h3Q{w1!kg(V2Ej5NMlW_DhMdQdIHUfYE3iCX`y zKB2NV7weBmEXYS8 zdJA?)=XB>Hm+S0&FF4srh*$`ZLbtWB-sGq_J{aB!+hxop!95{dFi)11{;p3%@Z)en z3Iv|Ba@I?Z5FH$3UO1QChbkOzBySjRyx(yX4ZoXnfC~gc@(4gYEJWEF>zKRTC@+1bZ7YtlcX+>%>^m}=B^iCx;R|H zkCSd9)rBie+kqT~2^&#vYPsG4Y6>FWk4;p8M!Q1XOrbA$6F*UQ`N2p{?FH#?erNG9 zYD1KV05da%^`dBa4z6=uQuBHj%5(q04taP<1uE@6AmzE;Hf-k)h}j8D4rUJWvBr9} z@47bmmG$;B7XXsL78k6XOz5t8Pjuc2b0?0L^ATtu&EtHS=|rMJ?!CK^u^ZoVmHe*J z(<_&+qBdhJCz5CH(c_y;2tmY}g+rq2`Xz@AukCsrkBJ>CskR~oi|o_Yz+kggb-x&v zaHX3ESUf}$B!0(CM;T%l>{dIcj7YNeVO?n(mID02ouW?RlkXTux2`D6bn@q`9fG~+ z?~H9_lv04Uu=mcE%K`+*Zjd~6|MC}srOz_2fg8bxQ3K1zX%BJSDhPf;iVqW{tzR!j zx8RH79q5C;rxa4VQh8AJxfN(|Q+S7MicUsi$i9P>`Y90R%ze&x{@(e$;)9secm;%w zU?4_dn+_E8`y$&uNHm#Gnuy$EIHx%!IS&3-+n*?!oic!PVUN?6ZaQl@^4=9 z-=f56#O&)`KQqdTXy*u+0td#+Ub9@(5~o~#w8izcS(jqq;)_{^X;bcQyp;Y3tMV?8 zNq<}pfN(!UmqgCDv!lj<$v#Q4MuoGJVqbZiRx9su(&U{1osvBqjCQO49*K(ATfU&g zsrGh{4=mO5Q5JJM&V#EvofEb{)BR+_d&sOKG{Wf!+pc*>h@BZNJ6DmokS=}ByZ*f? z!Me8`AaPtmmoOIr-0LRYG~hwDdiKVyJa69{yws(Ssf%v^9CrsqC zOW~T^?7T4gNgGFfUu>BZQP_#`P-2~o>SA7qtRCxy3ppO}lkN3~g4nNWCP1ycuRJru=dKA zen)kjds_dBA{{N8`koLsZNI(VI|yd4151)#&KV-jroHVIn?*dtR-^tnUIeP`oFO02`(2kI#u3l`ZD3yc_P)QfO}&x0bFqE+G;bqr zbj_xS)UM)8V!tGV{P|obtSE71E#vorLu|N?@2?es)q3B zQj9e`25}~9mngU3PLcR4fvQHC*2q+P(C@S9y7r{?ijgbT?JqkQ>=M4z^gZ$NdpN;;w7l)g8g3|6wLLCV>8B8V ztBau)oV%NP=seuAIDA#bm#|ed+Wo6#8CVyDKMJ#NqR{}{gmfG&6}7DxaKG_l`+aFV zEY0Rp6%1?>v_4ZGtDQAYQaLTYR#D|+)3oal^|}tXk#frS>OZjymc;PC4=qNzVwmO2 zvq0!KZ-X5~)@I?%h_sia24VU0jC+!U79Kj2c^lwEu0m>2IzIc!=a~s?b3qWdi)iga z{E}_CwVb~yA17FEN^DLv%+JtZHM2b8l(i^oEh{-Qp7#L2bN=zr6du#~%6er30|P(1K)+Le8Pyn)lh zweg2eUD=#?V4EcQDVf;IGLwVR!WugT+Aq#oirVI))q3t7p8@<|2wJQ>Kdu~NXeXv- z0cyYKyWTyBO!nVL=d22uTlkVTN3>y{!Xct3{|za{d6En^5otgyme4xPiCJ3McQua_ zGdXm|W@Q8Cod}|&U%n67jyV#HyEq}+a@)+&z*iV0tIx$>89LVRQoCJehWBz(DN4-Z zZ?Ry~S8oiFd={L4cLU;D`{CzNxaaVuNXk^NfG96(`eT zFt+;RaFM9T!UIZG=fd{WxB0LH2_c{EXK3Pk#X5d#uDes_@~YPJ4A_5`GG@KWU-r_U z?GUT9o$e;gue2AFDXF2TG|Kwa>e@C0J>EkP&6Aw02a9hNYV|7@CFp2dwq4d; z3{4}A2xNFHS&B_wFgnUMxS6qR=JoLpIEd4&;-rrbvi3yK0?t1dPhkl@0OC^rpob}^ zWwZyY(GEDoRwKkQ=gHd;GIF6AA!GUCu?RL&DtH|dY_iCB`m_eg`Ha<(pnvt% zHd!#Fv8CC~aotgfHfY^)I6a}tS6&Q8<#zlRX9FhNAx&Y*-#QwR zigLF_REY-L#d3!Z{i@0uejKU&w@D0iEkU2BcyX4&*>!IO;5ACshzu2eb zTJ3CmTLH$V%Y14XsIjG6c8OkJT63CcQVU`~iC26Lb}hht)m49`qX?rXX{41ZZQHl4 zLu_%X2>2o=_aGwOH|Q;>V4-eGQG8=>R1*i^C(ElwVCE`1@yM}ajhgadl|YeY@Kitd zUCK8*ctfoQrScXvnMctE^6d6a$j!?tK~D$>_S8WBp5jD1fmEQw@*&601Ub?9_>1RV zh@`)~?Mf7rltJr|C*PiINcxzpYStb36+~lbq^xdkn@B65f(|-qjRli%fy1Qt&y{_uRi5%aLsnnpZCW1-<*B zq%OL=dX?=wPU(>Ny#$aDqeLx)rR2g-nRa!iR+XBeFY7J+C#1STcruA5fjV)5zyllu zMr5*`TGd9j&2sry)>lql?%Ja;v5(yzAvSySjxwhPv9m=-b^y2^C852tjFZp#CbCu0 zJ5km~eW+-WXGIdOw$^$J0g%6yayH7QV*tW)P5$fvO(*-D`risCDfQyRB7##lcQ{rH z6=jp1H>J~MC^A0hlBF3z5$JPKN^ud0vEj;N?1Qm6*>7ZPYaGVV)13ej&AK~C73FZ=M^)Pn}E*w zgT1P{H+5PlL=G<^5!1I09+``mec-D%ArgCL*vMZK-+Wt(qEJ8$=6PooQRA6XR@SyzjgG}Tr|4#h$ zC*yPCNe50b;VFA}`Q;E!0jQ+ZD$}f$;kQh44P0Z}hz?4+ND7}Xj1_WICDP;%?Ms@I zryWiy?1Ok}pGl0B#m|=(>1M560GGoi7s635PKh;)ip__@9I}naRhLhdF}ZOlM8#sL z3(yn0%r80KP(Dz88yr9U3xayU`g#IOENc@@Uy;?UQSrNVz2onTakwV|xz3UogCXsM z5CGl;+q?4cP$ch%P#AG5OqI zgU{qGjDIAbqwv)kv;Y^$$B7QxZceDb& zMcdzJnY9x3lDp%l2W=V8yvVLA<1T7i*9|F$RL_or9=+QO8ssR8Q@DsS$qHUD2`Yr@ zVN2YWuh34LGbKIk=aPSh&{Hl+MEFUaGgOgzcU6}oFDD?JGvo$@$iOe(v2Zs-8tRP8 z3>?1L6(s$ILcdKo{h$WqJcdPHg)sZ}4y0$gf%a^dpv-(2|AlG}z~ZQrOkEJ2>=O7K z5Zw>}a`hLLH&>Qn4X(sf<=hj5U-e=;WxEzg;yK=$8tW;c|BTla7rtbAoM3-tBqQ`t z?+CGg7(n=dvJlTN@8Gb5r;(fJL@Vs-PT$c?#S-$kP;baGM$)&U70H`nJ;8LsO2EF# z-=Az5L%zYDF2$!RYf)6ql2LdylqGbafkZU>A~4Q}lLJ!WC8fp;oRHM$bZcI5fEb#L zTebE}?Uxi!u*=lDM#&trZm3lLd>Hq{8``zlxaBmmhU~DXs0#yQgWzowh#o>|iKaSf z4f!Q6SobL#;uY;U>`l>i{rimREfiNNouB%-O>&)}J`TJo7mN0uxnddfIT~oX%rQx* z_t`l8C6Kd5L^nbxQKYx)76yry_qD1_y5W4x!+wp@=OqI!2=Ny@I@yXAA_ZbR+!$@n zNKT>0LDee}iSm8SGwy<&cn#YH5jV2Nntq(hYzv(zu_&!ohM!gwE1v~~5%%ma%}<<} z!f8LzDD?S3w{B>Bj?KX6z}?9&pM6+Pjk?^?@9m0US@s3qwvg&=aLOFU-%4p$Xz&n5 zx0%Ytk4Q6Igr*Y>%OT5-3q=2NpcPI4Mx5mji2EACLZ~-#5-drq*T_vABNLKTQZyPp z)3}!vt`qBY{e#O`e-=>I>cax<;wu@(luXetxOm-Nz{()=vE_ zFQTWTP!hg#1ZitZaNd3z{AQH$fO~4IqG*X(gQ-7sAcnbsEyN57Ku?AU^?^QL4S01S zV6oeijVHPLK{+10^m>R(>M!kmrFH3L^g!Y)g%I*Bfr4Nag?DJ>A>T_UB|~NY;90N* z8a-%)i>Aap=X`=1AyU)f7K1XFSmBbAN6M!gL42`bk%!++L``gt^wLx0LiCUT@(@ud zAaFD=&%M;LAOU4@{%ljOJ1e=7U;1u`Hgo zZY%?)sw`+4k|df0zJAF&fleT}JhM4CdR7OYjF(zl)kxib=d<`xl7w7f+uw_NfR$($ z5DbxzL(4R&kqb)H@}=;WI+IX@a!}jIG}f{~ZqtE-VKq2?bs6RZB+;^1T&xh1LM0A5 zsb=&V2mRlsy9PKb`)t9lgO0u7VvE$t?XEmY&ZaMgf1t}~_;*Pwj3uYDf#*k5MC4r$ zzkyXD9+jokR8Jn`nL(xpUme;L*bfAT~!%VfwdRm;8kZ|DsD6AIGG~nEf>#vuXGs9vQR3{jTdvd087*nFkl&P|EGP&L=t`8bS@>W(>Hm!85G8BW=q?n`nYS|u>b#iVjZ^PVI_ z6SkP_5x-W3)tf5b3a-^6v0~6~*=r9es;rv^E_>&CqKgoQ3Ce}gvxGJ)5jy=w4?x|O zZ?7V!94ldfeBIldC=4}impn%`l}4K_CL;2s>3V8)Pr?QzFs{!~{O+Vf@O@z0^tQ{+ z^3mhN{h67~+WDS@=k9drT;~?Kft&rF8WVi(QWZf9ENiX}r*^9MSdK z;#-OM^V|DfKR5Zc2Um&qd8_^+9o;vsK}RB@R(}nL-9iK6^7Uf=qt5VSTNkfZ{U*dZ+zztRB9XVl&33j?|LB}_&WU*DgJuzmie@9k5Z+Zgz59Q_{!&8$E- zcERFN6oyQh6;+AC~Q$ZaJbxm)b1Gz=U?A zlG^1@(*27asw+pN+9r#wOZa=&`z;y7hVC1ApBsAQOag=CGg8ljs7Dy9PKFv;;`!II z#C^M|Kj?s=9LR&~yRNYtrskF~7#Jd(e_DY2?+GFrU7q&7<(7$E^JeokE*>|jJnH1$ zf5{|nG1!jEs_$3z&7M#De>$b{JW<`fJ_l(3eUL%@IJng4-fVF(RJqz>zH`ab-n!!D zc{kQxB6jvMiz`>BwScpB>5OR9k9%i{zUvde+uCjA*uBA4%zu3J_P`G8ncXxCmplWm zK>>Qde_yu_jDNN&{YA~dZz2KxVR^~7pAKtNX5E`UOq^?M;?4XN>}mR1JUb?uPwfVA{${Z9M*Y+9ix|FCF}3NzXPV5ZAS2zLiwT zYv6zDYdiYbo~MsrJr=X)L>M$F1|{}W*AWPTO);}3&fn$Y_n0`QBsBex#w*|S|Iv9$ z+@1*X$W2a^lSBzzs_z*MiiA)~A3aVbYhTHs_h|%jsPYP-xk-U4mzQ~QV^QBcsC9}& z9Qp;w_qAF`@&X&AhT=v3)dBSXV3Up-AH_>G!tHU)7M-GZ)Gc@?@T~nG^`06ET(53& zh^83!bF;k8T|HO$pA;Yz_`bMvx&_Qr?sTthRxID>ePgi;;{t;@xn!_-iUT*u}g{Ex@K<^jdq>2 zWb%gsvj52lQbRTWcHEMa3#3!q0|Cno+keo)9{O{%m{uhlBeLD){wpGtbYQu*OaIIq zC>6fqSbZJ}g1gnYMgM5V(dct9GXaGJ_%ymK4eH{v{%IcwJaZ)~@{5PXNCPcwO)jSo zYIk$ppR|!h_BYV6s%D=g3M6(8mu5qo@$=V4gdMt>q{xfup*^Y*+n;P>CNlf5YbF1w zKoiSI%;589x6OF7(}jr{u2LqGDy}?1Dj^V1!jb7~ zl&~;gIyzR@7KRolL z{|b!(dacva$T;sU-1`>pYqP~pl}z}8l~_`c`r##EhGMx_>meDGiw}@j+$CsacSBp8 zN@f&X3>l3{p&tQ(@!OHHk z@5orTf3)H1V)|#bTf#cQp220uIW*BD`2@7Wh{6RVf5>wEu63KII=E&{dFh)0=LCNR z0?e3Bb4Kzg=Q=_h0kP^%+fK_^KI&o5lsfc5dkGNJ?H92)Z^@+Q{cwMIQJS-b(Lpy0 z%Bm3arHVS!Zjq@k5EckebhtqWD1qpnBokH)`I+4NYxghAYlMzihKHoyfIh8p%ii08 zVC3W|O}zc^XVqI7TN^Ayq^(!qIoLCpO4<;yaom-XGeA}njaNeY3ckEMBo`X3zb^oSIZZ51*h;>qKV$}=s&OHxkh72-+NToOA!&o{{-9uZw`DgR=MI@QYWO*kI`nQw2;v}kg1<(X*dUk* z6bj2lY#rZ^fy9YC_Cb64iyn-Mv}?#({C7z?R5`LH0^Lx*Q2&lOI_QvpJ(zUe_wg8h!TNrvDBmT>0FfSBLHRTHVSm;q~BXldyuiK}GIVqyJ0#Iqp;|ed)6F*nE#M^?C#4oId3NIo2DostZC*8r2;X$bLl5N5+l(0eA1F zQn~Spa;%e@L$a>Z`}V`b1>C6?ypm8UiGMnBnT606Ni(Os9kg1USm(njqaEto0D28%sl-6^8U_+DS1aV0b-AdM^oZtWl0CWO6^;&4JCYt!L`CN&SN91h6_ST;9m z3|ft*WhX-<`u79H0(U!P!8I3ru#8k8Q&I)8zo@_P6^L>nvqjjyOkX zL(TBQcJTLG3%u3t!Y{O@dYA@gC3`+kaC=BA;1#)vMNO)K@aq^BB+Z3f17R$2DHv}z zBbrQuCHV|%SP!6-RZvGp7FBH`<9L${zT852tcpHfn}%2o&ZZ^VYu!o_cz1lbM_I0f z@1zxG1w;V3=|~@>mLq$khr`{4&g)Sebm2yj(rM53(7vOm$n%GvpX~%(wgnjuUKqoj z%FO(hkT!=M2E{0-Gk865o0}T3e#I^9wb=(iwUhEQXJiaIM;6;T)};A%exBpTqAv55}-tc%5vN#7o_`^MFou;lq-f~&f1$nL)1njB8p2;;&j2#-w*>Ivk% zyOI4gRM8y(mzzZK#mk)Tu-g)dF`DmCAq$2zfjO0WSNbz~CEq`{y+Ftfc?0|0iq@y- zK_Xm~mU^FMQ0DuWe5tX;K;rJ*8$h0M2SvOJmf=9ILLeK_9Pno|L}!qTv5+@Knn1wn zreCbz9&7Gx3yQ?H=3P-Zc#Dp^h~OJ%0$DXPX{aCR&hY1W!qnUov3cY(0q178lxLAI zHdP=zVI!QV&Bu3lxRW8wZ~-DTrS*cf0mv?3YSBd_WD3IWS$I??ZP0kn>R-?+NAJ-S z>qgGOHjdfcbUMX>(16QfWvlaAi8yC(lN8(aGfgjvMowuSWDm^&nui8|zfA-n-j%nJ zt-G&_Mh|-E&cZj+8`LAyJZ5reB(z%Jtql77W~+pmi?9xIk8bE&huvnMBb;MpD394du@yg>Jf#~+9#*xqOyb2;|cF;~?XVw)oMi3jz+)H}+`uJvrTw3EBbJu(fvax}AD zL&MlRn%;S3^;wXcN5m~>ezMPan~8W6Uk1|BU$$y_gvf_aEiNJ7ZRIE!eK| zhux}!-~4pn<6dk0e(fawMi`67Lxop-&{FQF8s5olT}r91}JAm<+@bi$_Xz%8#^%bL!8-8Y$EeSM%}0pecW&Nev

OB}#(hK&ZWj3s`n{7HrqjtxUSa`@>~z?#eOju(Nt1z7pe-=|v*U=Ekv- zvB>rJh)W)ALCqQR(daQ*!uj&;)>8ua32{)Bh?V|;Di~W#_Yw|bx5d{m*`?_=2jD2t z`SJ?v`+8pBDQ?^7(F}h|ql35_SF4RIz}n1;bo;S<2~S||*ug7PS2x`L^I#c}?Z>#} zkSMmzuWX2<9sEv{}ruJgU^VCR&2SE z;p;lUs6OB)Gwv+O-1kV$020>bKYQ^r5v(bd@qa0TKFB_9WdibISQ%MxH#Y+NH_0lf z87r3bDWM+hLjz<&dPC{bB^Sq+V7J|n{q%zpNx#PP3mNbO#ujUNGIyeTC$xjji_Kao zqXX@VI?Y5sW7@VO1SCZ)q@ieOFrv1FM|T@ax$wfWqUvmY(rDP+c%|_wcc*3?2C;HJ zOvUW$%=Az$rRhJ0;)x$Qa|P`_b|~ZVLn@PVZg~Ja@zUt~@Hrt+RVeN{?3qY|c<}wW zVgQ5s2%?T+!!Ks^$+d2jlaDEK6NEM#RoyP}0TfG&AXR|NWlwV3;&iY9%kr#~*QZ*Y z7#M8tfocarx&cDLiYb4S_=!la4Jsc!r6E40jeF>DpatoJD74|Vf-C92&g4SL=ci}^ zuLgcq+0@K)lslTv@O%6mBPO^adFvk1#1m~ebhsIL>X=p$J(dg9lm=I< z3G$e>0KU*b{&fIqLX@>h3)STrmLXNHuv(L?*Qp}kp7IKagPGVk2IybtvF_WOf;5&m zm5c|(KZkLKb0*j*!S4$YE3AVP+xCB_0y;VZ@g_)fBYbmd?Co^Mo@tt`Hj;V8zQf+y z!i}$F$ne(&S4NRQbB=QKZJ%k-@==ArfMkeMgtbX@j%#6pM*fI6zbrK�+w#YfYu9x==Sog@SmS=U`POT9?AfMG-ESqPBdf{8;3K+yCxUZ< zbK8c_uOYeCk0Dc3F7j(q+P!hfz)&tw=kbDx%ebgrlnO%y)4YJXsQK+1#i^}44Pzcb zA?;m+wO#NMh*^iY8Wg}%(bPi~y?d$^p7j0AHM)UUCb67(X#xuuxeBpGn|9}`guLaN z)Y1#z_m`kJNIxM&93Z>i1*!NTx&FcY$d|4rjlwf0>5rt*F4;i!Yx4yaGNK-qaA?)8mN| zME*?9{c)GPk{$sul_jd=)!g>wlHILk-PYBpMs~)hvwRaNaJ;5Ae}9X+G%M+deHChJo#HOHA3&mN7)~BBMP~Ceh%QKfprNg;N z$fpAGSW-ANT#=nwHZF$?x)mCRgGd``l_pWOt~?0~4ha<|k7h^w03S{9yvkGY7I*=c&@fz&@Sl@2aZQ_LRhM?C9@@)TwUKZQ%hq|y}Ac|fp1 zt}^PxvpFZ3Nz5?K?=g!7??3VFZudzh6NGVJS}57zTgvH=Yi?)mkfCL~w+LCI$#0ad zp+Ie76R?A7GpdQ_S}J4lSO)0S=w$ADFLo<3f-7+xVj8w^^=hOS41?NL9Df85e@dq9 zWJ3+b(8w4h58w7tcv#3rr^r2dlNpc*MSC9%zKNH4HyH8CULhA68(TaNu9Gqh|4BV= z44?cdaVr=%9AEmk3|ygV zXtj=LVmo(e^&7govvs3HX!Vg^$kI`gFS?7s`!0&u?W{c5%0Gs+~&!W@mLl~#>Tj)JHXIc*8H8umI0xMOeP_%FT>jOWN|Evn*g z<2d_^wxq$(cFZ=prmRp-a5At&1}Aq?9JjeTk%M1)DojHDI1v6DOq=|Bxa`yhIW$~O z5>$%tPpV`erbUJAK%dv$2EQGZ4wjVD#(@eJ9yCMzcCvh1=qk-jwn=QLU561ZaY*|~ z0jEzG;C;F3d`2bmA`;$@$=@uEK>cXSOy;H&A>(YV=E=yDO6DHFm=y~#zV+V9O02Qf z%=J^8U~FPs84=EI7{js<+iYLWBQD_G-up{;V7;(Ix3Vy!nhH7YB_EEY3aDqgsmf{S zV^_es=F8hg`~y}}X;@@irYz(h0T{QeN(B>y#RUvsX(2oWd%V6jo?^tSF;h6utvqru zv0EU*HUD)5KnZ|W2oLqW>*!yx>JOeQ&;yFLA9cY1FOne;+~${@{dWu{ z+W$5k;VEb()@^(ai~hWf=#M_RJ<7;Eg+Dp=N=QeCr!8#lO2G9=Ln)UzRhB@GMZzLF z{mM8Yume%!lYY{6;xAkf=oG+dnYxVY5)`YM0writXXOy9pm!ra_#6@y%)#;}tg$JQ zH~gOD${4Il*|Pt@s0UZJb7|=4Qquf~F)2y3q4*0r(i1Q$Ld^*nnVsnCZO}gh4K-f94z;@N@KPpW@8fpw2P&OokJyp!l_89P8{8|3j>pBR09$ zrGUJBZg~>te9jb_#|Ho0RLZLWB#)or>On9jMaIb($nK&(ylu8I$k~YR$7^XQC2>Ny zfiRo*NamGl(HKCHCnw)Fo*yE literal 0 HcmV?d00001 From 1b2fcf5a98f36dc74a236b589ad79e0dcd109587 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 08:16:51 -0700 Subject: [PATCH 018/138] Add author lines --- conf/czbiohub.config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index ff1523e..1c788b9 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -1,12 +1,19 @@ /* * ------------------------------------------------- - * Nextflow config file for UPPMAX (milou / irma) + * Nextflow config file for Chan Zuckerberg Biohub * ------------------------------------------------- * Defines reference genomes, using iGenome paths * Imported under the default 'standard' Nextflow * profile in nextflow.config */ + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + docker { enabled = true } From 9c0315addb194e2360c6fe76d53a9947d3866fab Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Fri, 31 May 2019 13:17:48 -0700 Subject: [PATCH 019/138] Use flat nongzipped files --- conf/czbiohub.config | 52 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/conf/czbiohub.config b/conf/czbiohub.config index 1c788b9..d51c6f2 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub.config @@ -47,59 +47,59 @@ params { // GENCODE GTF and fasta files genomes { 'GRCh38' { - fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa.gz" - gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/human/v29/GRCh38.p12.genome.fa" + gtf = "${params.gencode_base}/human/v29/gencode.vM19.annotation.gtf" } 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa.gz" - gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf.gz" + fasta = "${params.gencode_base}/mouse/vM19/GRCm38.p6.genome.fa" + gtf = "${params.gencode_base}/mouse/vM19/gencode.vM19.annotation.gtf" } } transgenes { 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa.gz" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf.gz" + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" } 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa.gz" - gtf = "${params.transgenes_base}/Cre/Cre.gtf.gz" + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" } 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa.gz" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf.gz" + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" } 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa.gz" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf.gz" + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" } 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa.gz" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf.gz" + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" } 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa.gz" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf.gz" + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" } 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa.gz" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf.gz" + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" } 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa.gz" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf.gz" + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" } 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa.gz" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf.gz" + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" } 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa.gz" - gtf = "${params.transgenes_base}/car-t/car-t.gtf.gz" + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" } 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa.gz" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf.gz" + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" } } } From 4491907f1461185b8be7e6906e9d7b583dc91fd3 Mon Sep 17 00:00:00 2001 From: Tobias Schraink Date: Tue, 4 Jun 2019 12:56:11 -0400 Subject: [PATCH 020/138] updated singularity version in conf/prince.config to reflect version installed on prince The prince admins changed the singularity version to 3.2.1 from 3.1.0, making the old version unavailable. To keep this config functional, I updated the version number for singularity. --- conf/prince.config | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 6ae4ee8..1c2ea2c 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -1,4 +1,6 @@ singularityDir = "$SCRATCH/singularity_images_nextflow" +singularityModule = "singularity/3.2.1" +squashfsModule = "squashfs/4.3" params { config_profile_description = """ @@ -17,8 +19,8 @@ singularity { process { beforeScript = """ - module load singularity/3.1.0 - module load squashfs/4.3 + module load $singularityModule + module load $squashfsModule """ .stripIndent() executor = 'slurm' From 3a481402b82debaee410afeb2b64d7ad5afd1039 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:07 -0700 Subject: [PATCH 021/138] Get czbiohub config to work --- conf/{czbiohub.config => czbiohub_aws.config} | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) rename conf/{czbiohub.config => czbiohub_aws.config} (95%) diff --git a/conf/czbiohub.config b/conf/czbiohub_aws.config similarity index 95% rename from conf/czbiohub.config rename to conf/czbiohub_aws.config index d51c6f2..baf4550 100644 --- a/conf/czbiohub.config +++ b/conf/czbiohub_aws.config @@ -20,8 +20,15 @@ docker { process { executor = 'awsbatch' + queue = 'nextflow' } +workDir = "s3://czb-nextflow/work" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + params { saveReference = true @@ -42,7 +49,6 @@ params { // AWS configurations awsregion = "us-west-2" awsqueue = "nextflow" - workDir = "s3://czb-nextflow/rnaseq/" // GENCODE GTF and fasta files genomes { From 268e411d10eefff821625e0f72346d52cf30842e Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:15 -0700 Subject: [PATCH 022/138] Add czbiohub_aws to list --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 7374fa0..0ca7fbe 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From ad7e26ea69388a906397c41da9363a91c76c2a1c Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 5 Jun 2019 14:42:24 -0700 Subject: [PATCH 023/138] Write out docs for czbiohub" git push --- docs/czbiohub.md | 141 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..6234fc2 --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,141 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. Make sure you have [aegea](https://github.com/czbiohub/codonboarding/blob/master/guides/aegea.md) installed to make launching AWS instances from the command line much easier. + +### 1. Launch the instance + +There is an Elastic Compute Cluster (EC2) Amazon machine image (AMI) set up with everything you need for Nextflow already installed. Its ID is `ami-091ec599f8e77734d` and can be launched from the command line with aegea like this: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-091ec599f8e77734d --subnet subnet-672e832e $USER-nextflow +``` + +For example: + +``` +aegea launch --iam-role S3fromEC2 -t t2.small --ami ami-0adcc973d6f458a1e --subnet subnet-672e832e olgabot-nextflow +``` + +### 2. Log into the instance + +Log into the instance with `aegea ssh`: + +``` +aegea ssh ec2-user@$USER-nextflow +``` + +For a concrete example: + +``` +aegea ssh ec2-user@olgabot-nextflow +``` + +### 3. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop and lost your connection. Start a new tmux session with `tmux new` + +``` +tmux new +``` + +Now you can run pipelines with abandon! + +### 4. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 5. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 6. If you lose connection, how do you re-attach the tmux session? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to your AWS EC2 instance. To reattach, use the command `tmux attach` and you should see your Nextflow output! + +``` +tmux attach +``` + + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From c7eed3fa4c92d92c8b5ef442d1b18df1256f4529 Mon Sep 17 00:00:00 2001 From: maxibor Date: Fri, 14 Jun 2019 19:16:20 +0200 Subject: [PATCH 024/138] fix queuesize scope and singularity --- conf/shh.config | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conf/shh.config b/conf/shh.config index 2b195a6..5cb3396 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -8,12 +8,16 @@ params { singularity { enabled = true autoMounts = true + runOptions = '-B /run/shm:/run/shm' cacheDir = "/projects1/singularity_scratch/cache/" } process { executor = 'slurm' queue = 'short' +} + +executor { queueSize = 16 } From f40ed194e3afb9d0c6ad0c511943c404ad690cd9 Mon Sep 17 00:00:00 2001 From: Szilveszter Juhos Date: Mon, 24 Jun 2019 09:54:37 +0200 Subject: [PATCH 025/138] Using correct reference location for munin --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 5d95f01..bca56f7 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -28,5 +28,5 @@ params { max_cpus = 16 max_time = 72.h // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/data0/btb/references/igenomes/' + igenomes_base = '/data1/references/igenomes/' } From acad9ac7e5eb850361a4797000199641ef29b3e9 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:50:36 -0700 Subject: [PATCH 026/138] Try to clean up czb update --- conf/czbiohub_aws.config | 123 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 conf/czbiohub_aws.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config new file mode 100644 index 0000000..5c47c12 --- /dev/null +++ b/conf/czbiohub_aws.config @@ -0,0 +1,123 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'nextflow' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + seq_center = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} From b0c67e0a9b45112c570b0a9856a4ed479190ab13 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:52:10 -0700 Subject: [PATCH 027/138] Add both seqCenter and seq_center --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 5c47c12..af7707a 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -38,7 +38,9 @@ params { max_cpus = 96 max_time = 240.h + // Compatible with multiple versions of rnaseq pipeline seq_center = "czbiohub" + seqCenter = "czbiohub" // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket // No final slash because it's added later From b82d68c009a4deb4c193bd440982381c4ba8556d Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 24 Jun 2019 17:55:01 -0700 Subject: [PATCH 028/138] Add docs --- docs/czbiohub.md | 125 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 docs/czbiohub.md diff --git a/docs/czbiohub.md b/docs/czbiohub.md new file mode 100644 index 0000000..5e98f4f --- /dev/null +++ b/docs/czbiohub.md @@ -0,0 +1,125 @@ +# nf-core/configs: CZ Biohub Configuration + +All nf-core pipelines have been successfully configured for use on the AWS Batch at the Chan Zuckerberg Biohub here. + +To use, run the pipeline with `-profile czbiohub_aws`. This will download and launch the [`czbiohub_aws.config`](../conf/czbiohub_aws.config) which has been pre-configured with a setup suitable for the AWS Batch. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Ask Olga (olga.botvinnik@czbiohub.org) if you have any questions! + +## Run the pipeline from a small AWS EC2 Instance + +The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure that the pipeline is successful, it will need to be run from a computer that has constant internet connection. Unfortunately for us, Biohub has spotty WiFi and even for short pipelines, it is highly recommended to run them from AWS. + +### 1. Start tmux + +[tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop. Start a new tmux session with `tmux new` and we'll name this session `nextflow`. + +``` +tmux new -n nextflow +``` + +Now you can run pipelines with abandon! + +### 2. Make a GitHub repo for your workflows (optional :) + +To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. + +``` +rnaseq: + nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ + +human_mouse_zebrafish: + nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ + --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv + + +merkin2012_aws: + nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ + -r olgabot/support-csv-directory-or-sra \ + -profile aws +``` + +In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: + +``` +make rnaseq +``` + +If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: + +``` +make human_mouse_zebrafish +``` + +Makefiles are a very useful way of storing longer commands with short mnemonic words. + + +Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: + +``` +git clone https://github.com/czbiohub/kh-workflows +``` + +Now both create and edit a `Makefile`: + +``` +cd +nano Makefile +``` + +Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. + + +``` +git add Makefile +git commit -m "Added makefile" +git push origin master +``` + + +### 3. Run your workflow!! + +Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ +``` + +### 4. If you lose connection, how do you restart the jobs? + +If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to AWS and may need to restart the jobs. To reattach, use the command `tmux attach` and you should see your Nextflow output! To get the named session, use: + +``` +tmux attach -n nextflow +``` + +To restart the jobs from where you left off, add the `-resume` flag to your `nextflow` command: + + +``` +nextflow run -profile czbiohub_aws nf-core/rnaseq \ + --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ + --genome GRCm38 \ + --outdir s3://olgabot-maca/nextflow-test/ \ + -resume +``` + +It's important that this command be re-run from the same directory as there is a "hidden" `.nextflow` folder that contains all the metadata and information about previous runs. + +## iGenomes specific configuration + +A local copy of the iGenomes resource has been made available on `s3://czbiohub-reference/igenomes` (in `us-west-2` region) so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. +You can do this by simply using the `--genome ` parameter. + +For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. + + +>NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From cfcb533b959d5fb2c4deb5750c6bb5645d5db009 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Tue, 25 Jun 2019 19:02:49 -0700 Subject: [PATCH 029/138] Add czbiohub_aws to list of configs" --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index dfcb4d2..0bfc474 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -16,6 +16,7 @@ profiles { ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From d1983f8acd15354160e60458355eee2cf75f9b56 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 26 Jun 2019 17:56:26 -0700 Subject: [PATCH 030/138] Use new and improved GENCODE+ERCC annotation --- conf/czbiohub_aws.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index af7707a..a89588b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -65,13 +65,13 @@ params { genomes { 'GRCh38' { fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gene_type.gtf" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" } } From 70b5205d153f4313bdf586837d0eb81e8b5f2607 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Thu, 27 Jun 2019 15:02:47 -0700 Subject: [PATCH 031/138] Add human salmon index and mouse star index --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index a89588b..bd0228d 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -68,11 +68,13 @@ params { gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" } 'GRCm38' { fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" } } From 6a2d2003f0fed491ccabc130c36e6301616983fa Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Mon, 1 Jul 2019 09:25:46 -0700 Subject: [PATCH 032/138] Add high priority AWS batch queue to CZBiohub AWS --- conf/czbiohub_aws.config | 2 +- conf/czbiohub_aws_highpriority.config | 127 ++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 conf/czbiohub_aws_highpriority.config diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index bd0228d..d20b405 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -20,7 +20,7 @@ docker { process { executor = 'awsbatch' - queue = 'nextflow' + queue = 'default-971039e0-830c-11e9-9e0b-02c5b84a8036' errorStrategy = 'ignore' } diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config new file mode 100644 index 0000000..2bf9a4d --- /dev/null +++ b/conf/czbiohub_aws_highpriority.config @@ -0,0 +1,127 @@ +/* + * ------------------------------------------------- + * Nextflow config file for Chan Zuckerberg Biohub + * ------------------------------------------------- + * Defines reference genomes, using iGenome paths + * Imported under the default 'standard' Nextflow + * profile in nextflow.config + */ + + //Profile config names for nf-core/configs + params { + config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' + config_profile_contact = 'Olga Botvinnik (@olgabot)' + config_profile_url = 'https://www.czbiohub.org/' + } + +docker { + enabled = true +} + +process { + executor = 'awsbatch' + queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' + errorStrategy = 'ignore' +} + +workDir = "s3://czb-nextflow/intermediates/" + +aws.region = 'us-west-2' +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' + +params { + saveReference = true + + // Largest SPOT instances available on AWS: https://ec2instances.info/ + max_memory = 1952.GB + max_cpus = 96 + max_time = 240.h + + // Compatible with multiple versions of rnaseq pipeline + seq_center = "czbiohub" + seqCenter = "czbiohub" + + // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + igenomes_base = "s3://czbiohub-reference/igenomes" + + // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket + // No final slash because it's added later + gencode_base = "s3://czbiohub-reference/gencode" + transgenes_base = "s3://czbiohub-reference/transgenes" + + // AWS configurations + awsregion = "us-west-2" + awsqueue = "nextflow" + + igenomesIgnore = true + + fc_extra_attributes = 'gene_name' + fc_group_features = 'gene_id' + fc_group_features_type = 'gene_type' + + // GENCODE GTF and fasta files + genomes { + 'GRCh38' { + fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" + gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" + star = "${params.gencode_base}/human/v30/STARIndex/" + salmon_index = "${params.gencode_base}/human/v30/salmon_index/" + } + 'GRCm38' { + fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" + gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" + transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" + start = "${params.gencode_base}/mouse/vM21/STARIndex/" + } + } + + transgenes { + 'ChR2' { + fasta = "${params.transgenes_base}/ChR2/ChR2.fa" + gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" + } + 'Cre' { + fasta = "${params.transgenes_base}/Cre/Cre.fa" + gtf = "${params.transgenes_base}/Cre/Cre.gtf" + } + 'ERCC' { + fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" + gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" + } + 'GCaMP6m' { + fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" + gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" + } + 'GFP' { + fasta = "${params.transgenes_base}/Gfp/Gfp.fa" + gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" + } + 'NpHR' { + fasta = "${params.transgenes_base}/NpHR/NpHR.fa" + gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" + } + 'RCaMP' { + fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" + gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" + } + 'RGECO' { + fasta = "${params.transgenes_base}/RGECO/RGECO.fa" + gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" + } + 'Tdtom' { + fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" + gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" + } + 'Car-T' { + fasta = "${params.transgenes_base}/car-t/car-t.fa" + gtf = "${params.transgenes_base}/car-t/car-t.gtf" + } + 'zsGreen' { + fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" + gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" + } + } +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 0bfc474..de5961c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,6 +17,7 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 45d36a150661b4f33a3847d345f17d41b74fa8ee Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 3 Jul 2019 09:21:16 +0200 Subject: [PATCH 033/138] Update cfc.config Add Weblog for CFC cluster --- conf/cfc.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/cfc.config b/conf/cfc.config index dd69647..2f9047d 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -14,6 +14,11 @@ process { executor = 'slurm' } +weblog{ + enabled = true + url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' +} + params { igenomes_base = '/nfsmounts/igenomes' max_memory = 60.GB From 917ec0863d5e427940ec8a19ecdc258df786aef1 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:03 -0700 Subject: [PATCH 034/138] add default trimming --- conf/czbiohub_aws.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index d20b405..71d1a9b 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -61,6 +61,8 @@ params { fc_group_features = 'gene_id' fc_group_features_type = 'gene_type' + trim_pattern = '_+S\\d+' + // GENCODE GTF and fasta files genomes { 'GRCh38' { From 069963ff8a5ce9f6cad3a7be71e44782f3417ee4 Mon Sep 17 00:00:00 2001 From: Olga Botvinnik Date: Wed, 3 Jul 2019 16:01:47 -0700 Subject: [PATCH 035/138] Make czbiohub_aws_highpriority only set the AWS Batch queue and nothing else --- conf/czbiohub_aws_highpriority.config | 115 -------------------------- nfcore_custom.config | 5 +- 2 files changed, 4 insertions(+), 116 deletions(-) diff --git a/conf/czbiohub_aws_highpriority.config b/conf/czbiohub_aws_highpriority.config index 2bf9a4d..5ab796a 100644 --- a/conf/czbiohub_aws_highpriority.config +++ b/conf/czbiohub_aws_highpriority.config @@ -7,121 +7,6 @@ * profile in nextflow.config */ - //Profile config names for nf-core/configs - params { - config_profile_description = 'Chan Zuckerberg Biohub AWS Batch profile provided by nf-core/configs.' - config_profile_contact = 'Olga Botvinnik (@olgabot)' - config_profile_url = 'https://www.czbiohub.org/' - } - -docker { - enabled = true -} - process { - executor = 'awsbatch' queue = 'highpriority-971039e0-830c-11e9-9e0b-02c5b84a8036' - errorStrategy = 'ignore' -} - -workDir = "s3://czb-nextflow/intermediates/" - -aws.region = 'us-west-2' -executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' - -params { - saveReference = true - - // Largest SPOT instances available on AWS: https://ec2instances.info/ - max_memory = 1952.GB - max_cpus = 96 - max_time = 240.h - - // Compatible with multiple versions of rnaseq pipeline - seq_center = "czbiohub" - seqCenter = "czbiohub" - - // illumina iGenomes reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - igenomes_base = "s3://czbiohub-reference/igenomes" - - // GENCODE (human + mouse) reference file paths on CZ Biohub reference s3 bucket - // No final slash because it's added later - gencode_base = "s3://czbiohub-reference/gencode" - transgenes_base = "s3://czbiohub-reference/transgenes" - - // AWS configurations - awsregion = "us-west-2" - awsqueue = "nextflow" - - igenomesIgnore = true - - fc_extra_attributes = 'gene_name' - fc_group_features = 'gene_id' - fc_group_features_type = 'gene_type' - - // GENCODE GTF and fasta files - genomes { - 'GRCh38' { - fasta = "${params.gencode_base}/human/v30/GRCh38.p12.genome.ERCC92.fa" - gtf = "${params.gencode_base}/human/v30/gencode.v30.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/human/v30/gencode.v30.transcripts.ERCC92.fa" - star = "${params.gencode_base}/human/v30/STARIndex/" - salmon_index = "${params.gencode_base}/human/v30/salmon_index/" - } - 'GRCm38' { - fasta = "${params.gencode_base}/mouse/vM21/GRCm38.p6.genome.ERCC92.fa" - gtf = "${params.gencode_base}/mouse/vM21/gencode.vM21.annotation.ERCC92.gtf" - transcript_fasta = "${params.gencode_base}/mouse/vM21/gencode.vM21.transcripts.ERCC92.fa" - start = "${params.gencode_base}/mouse/vM21/STARIndex/" - } - } - - transgenes { - 'ChR2' { - fasta = "${params.transgenes_base}/ChR2/ChR2.fa" - gtf = "${params.transgenes_base}/ChR2/ChR2.gtf" - } - 'Cre' { - fasta = "${params.transgenes_base}/Cre/Cre.fa" - gtf = "${params.transgenes_base}/Cre/Cre.gtf" - } - 'ERCC' { - fasta = "${params.transgenes_base}/ERCC92/ERCC92.fa" - gtf = "${params.transgenes_base}/ERCC92/ERCC92.gtf" - } - 'GCaMP6m' { - fasta = "${params.transgenes_base}/GCaMP6m/GCaMP6m.fa" - gtf = "${params.transgenes_base}/GCaMP6m/GCaMP6m.gtf" - } - 'GFP' { - fasta = "${params.transgenes_base}/Gfp/Gfp.fa" - gtf = "${params.transgenes_base}/Gfp/Gfp.gtf" - } - 'NpHR' { - fasta = "${params.transgenes_base}/NpHR/NpHR.fa" - gtf = "${params.transgenes_base}/NpHR/NpHR.gtf" - } - 'RCaMP' { - fasta = "${params.transgenes_base}/RCaMP/RCaMP.fa" - gtf = "${params.transgenes_base}/RCaMP/RCaMP.gtf" - } - 'RGECO' { - fasta = "${params.transgenes_base}/RGECO/RGECO.fa" - gtf = "${params.transgenes_base}/RGECO/RGECO.gtf" - } - 'Tdtom' { - fasta = "${params.transgenes_base}/Tdtom/Tdtom.fa" - gtf = "${params.transgenes_base}/Tdtom/Tdtom.gtf" - } - 'Car-T' { - fasta = "${params.transgenes_base}/car-t/car-t.fa" - gtf = "${params.transgenes_base}/car-t/car-t.gtf" - } - 'zsGreen' { - fasta = "${params.transgenes_base}/zsGreen/zsGreen.fa" - gtf = "${params.transgenes_base}/zsGreen/zsGreen.gtf" - } - } } diff --git a/nfcore_custom.config b/nfcore_custom.config index de5961c..72dca30 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -17,7 +17,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { + includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" + } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From cb507177d5ceca20722bea2acda201c040d9300b Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:03:37 +0200 Subject: [PATCH 036/138] Updating CCGA config file --- conf/ccga.config | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conf/ccga.config b/conf/ccga.config index f29db9c..b9130e0 100644 --- a/conf/ccga.config +++ b/conf/ccga.config @@ -2,15 +2,20 @@ params { config_profile_description = 'CCGA cluster profile provided by nf-core/configs.' config_profile_contact = 'Marc Hoeppner (@marchoeppner)' - config_profile_url = 'https://www.ikmb.uni-kiel.de/' + config_profile_url = 'https://www.ccga.uni-kiel.de/' } /* * ------------------------------------------------- - * Nextflow config file with environment modules for RZCluster in Kiel + * Nextflow config file for CCGA cluster in Kiel * ------------------------------------------------- */ +singularity { + enabled = true + runOptions = "-B /ifs -B /scratch -B /work_beegfs" +} + executor { queueSize=100 } @@ -29,4 +34,7 @@ params { // illumina iGenomes reference file paths on RZCluster igenomes_base = '/ifs/data/nfs_share/ikmb_repository/references/iGenomes/references/' saveReference = true + max_memory = 128.GB + max_cpus = 16 + max_time = 120.h } From d66bf4e4786799301846b34dfd9602d17c4ce3ce Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:28:18 +0200 Subject: [PATCH 037/138] Updating CCGA documentation --- docs/ccga.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 docs/ccga.md diff --git a/docs/ccga.md b/docs/ccga.md new file mode 100644 index 0000000..4a38bd9 --- /dev/null +++ b/docs/ccga.md @@ -0,0 +1,18 @@ +# nf-core/configs: CCGA Configuration + +Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. + +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load IKMB +module load Java/1.8.0 +module load Nextflow +module load singularity3.1.0 +``` + +>NB: Access to the CCGA cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From fe0153befd41e7cd054a398d64dac7da89f1f45d Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Fri, 5 Jul 2019 08:30:02 +0200 Subject: [PATCH 038/138] Updating CCGA documentation --- docs/ccga.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga.md b/docs/ccga.md index 4a38bd9..1157cb3 100644 --- a/docs/ccga.md +++ b/docs/ccga.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. -To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the BINAC cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on the cluster. You can do this by issuing the commands below: From 9106559b2f319c0f78a021bc28abd714ab957c40 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 15 Jul 2019 17:53:11 +0200 Subject: [PATCH 039/138] Update cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 2f9047d..6fc3b26 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -16,7 +16,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/workflows' + url = 'http://services.qbic.uni-tuebingen.de:8080/workflowservice/workflows' } params { From 1b30c5a4d24301ad1100e305f4ad1885b9f22b3f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:02 +0200 Subject: [PATCH 040/138] Add the Pasteur profile --- conf/pasteur.config | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 conf/pasteur.config diff --git a/conf/pasteur.config b/conf/pasteur.config new file mode 100644 index 0000000..03971dc --- /dev/null +++ b/conf/pasteur.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Institut Pasteur HPC cluster profile' + config_profile_contact = 'Remi Planel (@rplanel)' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /local/scratch:/tmp' +} + +process { + executor = 'slurm' + clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } +} + +params { + igenomesIgnore = true + max_memory = 256.GB + max_cpus = 28 + max_time = 24.h +} From 1cb132a4572dc85bd7478bdb881cb1934e0cdf0f Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:32:25 +0200 Subject: [PATCH 041/138] Define a profile for the Pasteur config --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..a9c0729 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,6 +33,7 @@ profiles { uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From fbb33fde63700e1f76852ad7df2780bba4b7ec6a Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Wed, 24 Jul 2019 11:43:08 +0200 Subject: [PATCH 042/138] Details --- conf/pasteur.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 03971dc..2548d3f 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -2,7 +2,7 @@ params { config_profile_description = 'The Institut Pasteur HPC cluster profile' config_profile_contact = 'Remi Planel (@rplanel)' - config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster/' + config_profile_url = 'https://research.pasteur.fr/en/service/tars-cluster' } singularity { From 71bc98afaef852562d5664d7861cc31cb870db79 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:06:29 +0200 Subject: [PATCH 043/138] Adding config file for CCGA dx cluster --- docs/ccga_dx.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 docs/ccga_dx.md diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md new file mode 100644 index 0000000..62bf29c --- /dev/null +++ b/docs/ccga_dx.md @@ -0,0 +1,9 @@ +# nf-core/configs: CCGA DX Configuration + +Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. + +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to have Nextflow installed. + +>NB: Access to the CCGA DX cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). From b31db18dfb0b790aecc81c3685a5cc30a57fccec Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:08:07 +0200 Subject: [PATCH 044/138] Adding config file for CCGA dx cluster --- README.md | 1 + conf/ccga_dx.config | 37 +++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 1 + 3 files changed, 39 insertions(+) create mode 100644 conf/ccga_dx.config diff --git a/README.md b/README.md index 73b01be..d865c89 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) +* [CCGA-DX]/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config new file mode 100644 index 0000000..0af95e7 --- /dev/null +++ b/conf/ccga_dx.config @@ -0,0 +1,37 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'CCGA DX cluster profile provided by nf-core/configs.' + config_profile_contact = 'Marc Hoeppner (@marchoeppner)' + config_profile_url = 'https://www.ccga.uni-kiel.de/' +} + +/* + * ------------------------------------------------- + * Nextflow config file for CCGA cluster in Kiel + * ------------------------------------------------- + */ + +singularity { + enabled = true +} + +executor { + queueSize=100 +} + +process { + + // Global process config + executor = 'slurm' + queue = 'htc' + +} + +params { + // illumina iGenomes reference file paths on DX Cluster + igenomes_base = '/mnt/ld_ng_out/sukmb352/references/iGenomes/references/' + saveReference = true + max_memory = 250.GB + max_cpus = 20 + max_time = 240.h +} diff --git a/nfcore_custom.config b/nfcore_custom.config index 41b0ade..d4ae35b 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -14,6 +14,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } + ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } From 7d5de651aa238c77f6bee3347622d34c1aa8a942 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:08 +0200 Subject: [PATCH 045/138] Update README.md Co-Authored-By: Harshil Patel --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d865c89..d7b43fb 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Currently documentation is available for the following clusters: * [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) * [CCGA](docs/ccga.md) -* [CCGA-DX]/docs/ccga_dx.md) +* [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) * [GIS](docs/gis.md) From b74f020f14d3254d67d85edaa2a3c8d20dfea389 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Mon, 29 Jul 2019 12:45:30 +0200 Subject: [PATCH 046/138] Update docs/ccga_dx.md Co-Authored-By: Harshil Patel --- docs/ccga_dx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md index 62bf29c..0cea81e 100644 --- a/docs/ccga_dx.md +++ b/docs/ccga_dx.md @@ -2,7 +2,7 @@ Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. -To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga_dx.config`](../conf/ccga_dx.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Before running the pipeline you will need to have Nextflow installed. From 96ea16ddbd9082f630c4836d2cb032542c15151c Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:14:18 +0200 Subject: [PATCH 047/138] Ignore vscode workspace --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 07c0144..8aa0735 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ work/ data/ results/ .DS_Store +*.code-workspace \ No newline at end of file From b4cffe0455d1523ad179fd0d125857511e722216 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:07 +0200 Subject: [PATCH 048/138] Remove buggy cluster option --- conf/pasteur.config | 1 - 1 file changed, 1 deletion(-) diff --git a/conf/pasteur.config b/conf/pasteur.config index 2548d3f..1d95131 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -13,7 +13,6 @@ singularity { process { executor = 'slurm' - clusterOptions = { "-A $params.project ${params.clusterOptions ?: ''}" } } params { From 6f264ac2caca9429e27e72c1347d96dfdfcd9457 Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:15:30 +0200 Subject: [PATCH 049/138] Add pasteur profile doc --- docs/pasteur.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/pasteur.md diff --git a/docs/pasteur.md b/docs/pasteur.md new file mode 100644 index 0000000..153f4d2 --- /dev/null +++ b/docs/pasteur.md @@ -0,0 +1,54 @@ +# nf-core/configs: Institut Pasteur Configuration + +All nf-core pipelines have been successfully configured for use on the tars cluster at the Institut Pasteur. + +To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + + + +## Running the workflow on the Pasteur cluster + +>Nextflow is not installed by default on the Pasteur cluster. +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + + +>Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. +The compute nodes don't have access to internet so you need to run it offline. To do that: +1. Create a virtualenv to install nf-core in order to get the workflow +```bash +module purge +module load Python/3.6.0 +module load java +module load singularity +cd /path/to/nf-core/workflows +virtualenv .venv -p python3 +. .venv/bin/activate +``` +2. Install nf-core: [here](https://nf-co.re/tools#installation) +3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) +5. Run nextflow on a compute node: +```bash +# create a terminal +tmux + +# Get a compute node +salloc + +# Load the dependencies if not done before +module purge +module load java +module load singularity + +# Run nextflow workflow +nextflow run \\ +/path/to/pipeline-dir/from/step/2/workflow \\ +-resume +-profile pasteur \\ +-with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ +--email my-email@pasteur.fr \\ +--custom_config_base /path/to/configs/from/step/4/ \\ +-c my-specific.config +... +``` From 64e78a3dc15ef11224b8da0bf7ed64b9c30e816b Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 15:17:25 +0200 Subject: [PATCH 050/138] Update the doc --- docs/pasteur.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 153f4d2..676199d 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -43,7 +43,7 @@ module load singularity # Run nextflow workflow nextflow run \\ -/path/to/pipeline-dir/from/step/2/workflow \\ +/path/to/pipeline-dir/from/step/3/workflow \\ -resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ From f9a2baf6143fec9325cd8fea10eb4dcdd478470e Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:10:33 +0200 Subject: [PATCH 051/138] Add link to Pasteur doc --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 73b01be..edab0b5 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ Currently documentation is available for the following clusters: * [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) * [UZH](docs/uzh.md) +* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` From be28a34839f795ec9cff95102bc1b26d1ca231ae Mon Sep 17 00:00:00 2001 From: Remi Planel Date: Tue, 30 Jul 2019 16:12:30 +0200 Subject: [PATCH 052/138] Updat Pasteur doc --- docs/pasteur.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/pasteur.md b/docs/pasteur.md index 676199d..554761b 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -8,14 +8,16 @@ To use, run the pipeline with `-profile pasteur`. This will download and launch ## Running the workflow on the Pasteur cluster ->Nextflow is not installed by default on the Pasteur cluster. +Nextflow is not installed by default on the Pasteur cluster. - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) ->Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. -The compute nodes don't have access to internet so you need to run it offline. To do that: -1. Create a virtualenv to install nf-core in order to get the workflow +The compute nodes don't have access to internet so you need to run it offline. + +To do that: +1. Create a virtualenv to install nf-core ```bash module purge module load Python/3.6.0 From 29a141adca47d2b7908b6d29b0d0eccfe051a2be Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 16:54:58 +0200 Subject: [PATCH 053/138] Sort it --- README.md | 2 +- nfcore_custom.config | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4b4d41..af4aad1 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Currently documentation is available for the following clusters: * [HEBBE](docs/hebbe.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) +* [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) @@ -100,7 +101,6 @@ Currently documentation is available for the following clusters: * [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) * [UZH](docs/uzh.md) -* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` diff --git a/nfcore_custom.config b/nfcore_custom.config index ec40134..469d6af 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -13,6 +13,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } + bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } @@ -27,15 +28,14 @@ profiles { hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } - pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From 4e48d4cc62b8f4b7415867c2377534f0e9a17c74 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:00 +0200 Subject: [PATCH 054/138] Fix remaining points by drpatelh --- README.md | 4 ++-- docs/pasteur.md | 34 ++++++++++++++++++---------------- nfcore_custom.config | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index af4aad1..5323165 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,8 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: * [AQUILA](docs/aquila.md) -* [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) +* [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) @@ -98,8 +98,8 @@ Currently documentation is available for the following clusters: * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) +* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` diff --git a/docs/pasteur.md b/docs/pasteur.md index 554761b..6e2d825 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,36 +4,38 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - - ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. -- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) +- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. The compute nodes don't have access to internet so you need to run it offline. To do that: + 1. Create a virtualenv to install nf-core -```bash -module purge -module load Python/3.6.0 -module load java -module load singularity -cd /path/to/nf-core/workflows -virtualenv .venv -p python3 -. .venv/bin/activate -``` + + ```bash + module purge + module load Python/3.6.0 + module load java + module load singularity + cd /path/to/nf-core/workflows + virtualenv .venv -p python3 + . .venv/bin/activate + ``` + 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) -4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) -5. Run nextflow on a compute node: +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/configs#offline-usage) +5. Run nextflow on a compute node: + ```bash # create a terminal -tmux +tmux # Get a compute node salloc @@ -46,7 +48,7 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ diff --git a/nfcore_custom.config b/nfcore_custom.config index 469d6af..69bb8f3 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,8 +33,8 @@ profiles { prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 82ffedf895ad9d1afa28307fe406d64eb770844a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:36 +0200 Subject: [PATCH 055/138] Drop aquila --- docs/aquila.md | 8 -------- nfcore_custom.config | 1 - 2 files changed, 9 deletions(-) delete mode 100644 docs/aquila.md diff --git a/docs/aquila.md b/docs/aquila.md deleted file mode 100644 index 3e0575e..0000000 --- a/docs/aquila.md +++ /dev/null @@ -1,8 +0,0 @@ -# nf-core/configs: Aquila Configuration - -All nf-core pipelines have been successfully configured for use on the Aquila cluster at A*STAR Singapore. - -To use, run the pipeline with `-profile aquila`. This will download and launch the [`aquila.config`](../conf/aquila.config) which has been pre-configured with a setup suitable for the Aquila cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - ->NB: You will need an account to use the HPC cluster Aquila in order to run the pipeline. If in doubt contact IT. ->NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 69bb8f3..259b048 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,7 +12,6 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { - aquila { includeConfig "${params.custom_config_base}/conf/aquila.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } From d9930b053c3cd91c3cfa20c0bb556cf3cd460346 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:53:45 +0200 Subject: [PATCH 056/138] Drop aquila --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 5323165..c32df8c 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: -* [AQUILA](docs/aquila.md) * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) From ead80fb9d096820324a836cf4b1bafa0ebe2b41c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 30 Jul 2019 18:54:38 +0200 Subject: [PATCH 057/138] Drop config --- conf/aquila.config | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 conf/aquila.config diff --git a/conf/aquila.config b/conf/aquila.config deleted file mode 100644 index 05035d4..0000000 --- a/conf/aquila.config +++ /dev/null @@ -1,26 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'A*STAR Aquila cluster profile provided by nf-core/configs.' - config_profile_contact = 'Andreas Wilm (@andreas-wilm)' - config_profile_url = 'https://www.a-star.edu.sg/' -} - -singularity { - enabled = true -} - -process { - beforeScript = { 'source /mnt/projects/rpd/rc/init.2017-04; module load miniconda3; set +u; source activate ' + params.container.replaceAll(/[\/:]/, '-').replaceAll(/nfcore/, 'nf-core') + "; set -u;" } - executor = 'sge' - clusterOptions = { "-l mem_free=" + task.memory.toString().replaceAll(/[\sB]/,'') } - penv = 'OpenMP' -} - -params { - igenomes_base = '/nfsmounts/igenomes' - max_memory = 128.GB - max_cpus = 28 - max_time = 48.h -} - - From 6298f78bf40d4f2b66f7b316e30ae07a9c7c0b20 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 21:33:47 +0100 Subject: [PATCH 058/138] .travis.yml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 918a846..c8db01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work - - NXF_VER='' # Plus: get the latest NF version and check, that it works + - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config From 840a97ed68b589871ee7e688040ab1b960f4e7bd Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 21:52:16 +0100 Subject: [PATCH 059/138] Clean up and alphabetasize --- README.md | 10 ++++++---- nfcore_custom.config | 8 ++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 675a95e..657b226 100644 --- a/README.md +++ b/README.md @@ -82,24 +82,26 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs Currently documentation is available for the following clusters: -* [BINAC](docs/binac.md) * [BIGPURPLE](docs/bigpurple.md) +* [BINAC](docs/binac.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) * [CRICK](docs/crick.md) +* [CZBIOHUB_AWS](docs/czbiohub.md) +* [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) +* [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) * [PRINCE](docs/prince.md) * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UPPMAX](docs/uppmax.md) +* [UPPMAX-DEVEL](docs/uppmax-devel.md) * [UZH](docs/uzh.md) -* [PASTEUR](docs/pasteur.md) ### Uploading to `nf-core/configs` @@ -111,4 +113,4 @@ We will be notified automatically when you have created your pull request, and p ## Help -If you have any questions or issues please send us a message on [Slack](https://nf-core-invite.herokuapp.com/). +If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack). diff --git a/nfcore_custom.config b/nfcore_custom.config index 5dd8ee2..2800c1d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } @@ -21,20 +22,19 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" - } + } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } + pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } - pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } } // If user hostnames contain one of these substring and they are From daf1b9dcd259659d42753624b537310207880a75 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:09:30 +0100 Subject: [PATCH 060/138] Fix type in uppmax-devel --- README.md | 2 +- conf/{uppmax-devel.config => uppmax_devel.config} | 0 nfcore_custom.config | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename conf/{uppmax-devel.config => uppmax_devel.config} (100%) diff --git a/README.md b/README.md index 657b226..059aa30 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Currently documentation is available for the following clusters: * [SHH](docs/shh.md) * [UCT_HEX](docs/uct_hex.md) * [UPPMAX](docs/uppmax.md) -* [UPPMAX-DEVEL](docs/uppmax-devel.md) +* [UPPMAX_DEVEL](docs/uppmax.md) * [UZH](docs/uzh.md) ### Uploading to `nf-core/configs` diff --git a/conf/uppmax-devel.config b/conf/uppmax_devel.config similarity index 100% rename from conf/uppmax-devel.config rename to conf/uppmax_devel.config diff --git a/nfcore_custom.config b/nfcore_custom.config index 2800c1d..2ae18df 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -32,7 +32,7 @@ profiles { prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax-devel.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 8be28979617ff6a313c923488e5d7fd2dcb0ca68 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:27:37 +0100 Subject: [PATCH 061/138] Remove problematic configs --- nfcore_custom.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 2ae18df..37cd5f1 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,11 +18,6 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { - includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" - includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" - } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 70919d0dd30661bde3c64272785ad2f10afd5bf2 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:28:03 +0100 Subject: [PATCH 062/138] Remove more configs --- nfcore_custom.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 37cd5f1..c19ed92 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -19,12 +19,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } - hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } - prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } From a496bfaeef0eade7f157d8be7f1aa260d4879eab Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:33:49 +0100 Subject: [PATCH 063/138] Remove uppmax_devel --- nfcore_custom.config | 1 - 1 file changed, 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index c19ed92..0578add 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -25,7 +25,6 @@ profiles { phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 0f6802a7c89705ab61f39dc4170a5f143ff16602 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:37:11 +0100 Subject: [PATCH 064/138] Add hebbe back in --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 0578add..4b79495 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -19,6 +19,7 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } + hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } From f3e61fdd66e1b94c37a9dd42864827d92914cd6e Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:40:43 +0100 Subject: [PATCH 065/138] Add prince back in --- nfcore_custom.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 4b79495..943429a 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -24,6 +24,7 @@ profiles { munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } + prince { includeConfig "${params.custom_config_base}/conf/prince.config" } shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } From eb0e89eeaac958b2928558aee106e9a5a7a0a779 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:44:54 +0100 Subject: [PATCH 066/138] Remove SCRATCH variable --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8db01c..79de276 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. + - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config From 0326b5603897b0713cff5dee397c837f188b2b46 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:48:22 +0100 Subject: [PATCH 067/138] Add czbiohub back in --- .travis.yml | 4 ++-- nfcore_custom.config | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 79de276..c8db01c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,8 @@ install: - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests env: - - NXF_VER='18.10.1' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. + - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. + - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. script: # Run the pipeline with the test profile and test remote config diff --git a/nfcore_custom.config b/nfcore_custom.config index 943429a..6d21f5d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,6 +18,7 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } From 7ee70fb742696080266dc6cadde7e91ba66252ca Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 22:52:20 +0100 Subject: [PATCH 068/138] Add in uppmax-devel --- nfcore_custom.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 6d21f5d..eb0abf4 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,7 +18,7 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } @@ -29,6 +29,7 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From a692ded933e3871a07eda1ceee2db51138801905 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 23:30:51 +0100 Subject: [PATCH 069/138] Remove workDir in czbiohub --- conf/czbiohub_aws.config | 2 +- nfcore_custom.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 1dd8da2..576e49e 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -24,7 +24,7 @@ process { errorStrategy = 'ignore' } -workDir = "s3://czb-nextflow/intermediates/" +//workDir = "s3://czb-nextflow/intermediates/" aws.region = 'us-west-2' executor.awscli = '/home/ec2-user/miniconda/bin/aws' diff --git a/nfcore_custom.config b/nfcore_custom.config index eb0abf4..4180c1d 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -18,7 +18,8 @@ profiles { ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } - + czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } @@ -29,7 +30,6 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From e0c684ee8f1bc7be315659ba582737b15b71ee53 Mon Sep 17 00:00:00 2001 From: drpatelh Date: Wed, 31 Jul 2019 23:39:42 +0100 Subject: [PATCH 070/138] Aarrrgh pulling configs from master --- conf/czbiohub_aws.config | 2 +- nfcore_custom.config | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 576e49e..1dd8da2 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -24,7 +24,7 @@ process { errorStrategy = 'ignore' } -//workDir = "s3://czb-nextflow/intermediates/" +workDir = "s3://czb-nextflow/intermediates/" aws.region = 'us-west-2' executor.awscli = '/home/ec2-user/miniconda/bin/aws' diff --git a/nfcore_custom.config b/nfcore_custom.config index 4180c1d..2d4c183 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -30,6 +30,7 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 91a5e64e1cd0ab7944a3340617f457fe7da5aecc Mon Sep 17 00:00:00 2001 From: drpatelh Date: Thu, 1 Aug 2019 09:42:50 +0100 Subject: [PATCH 071/138] Ignore czbiohub_aws --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c8db01c..4985029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,5 @@ script: grep "{.*includeConfig.*[a-z]*\.config\"" ${TRAVIS_BUILD_DIR}/nfcore_custom.config | \ tr -s ' ' | \ cut -d " " -f 2 | \ + grep -v "czbiohub_aws" | \ xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} From c583aa0233bf5fd5769db4db3046fd66a5fb99e8 Mon Sep 17 00:00:00 2001 From: Martin Proks Date: Mon, 5 Aug 2019 11:19:06 +0200 Subject: [PATCH 072/138] Added kraken config (#15) * Added kraken template * docs: updated some info * fix: updated igenomes_base path --- conf/kraken.config | 24 ++++++++++++++++++++++++ docs/kraken.md | 10 ++++++++++ 2 files changed, 34 insertions(+) create mode 100644 conf/kraken.config create mode 100644 docs/kraken.md diff --git a/conf/kraken.config b/conf/kraken.config new file mode 100644 index 0000000..59417e9 --- /dev/null +++ b/conf/kraken.config @@ -0,0 +1,24 @@ +//Profile config names for nf-core/configs +params { + config_profile_name = 'KRAKEN' + config_profile_description = 'Jenkins cluster provided by nf-core/configs.' + config_profile_contact = 'Maxime Garcia or Johannes Alneberg' + config_profile_url = 'kraken.dyn.scilifelab.se' +} + +process { + executor = 'local' +} + +docker { + enabled = true + mountFlags = 'z' + fixOwnership = true +} + +params { + max_memory = 60.GB + max_cpus = 16 + max_time = 72.h + igenomes_base = '/share/igenomes/' +} diff --git a/docs/kraken.md b/docs/kraken.md new file mode 100644 index 0000000..9426087 --- /dev/null +++ b/docs/kraken.md @@ -0,0 +1,10 @@ +# nf-core/configs: KRAKEN Configuration + +This profile can be **only** combined with `jenkins.config`. It is used for +testing pipeline with real data on **in-house** cluster located at SciLifeLab. + +To use, run the pipeline with `-profile kraken`. This will download and launch +the [`kraken.config`](../conf/kraken.config) which has been pre-configured to +test the pipeline using `docker` by default. + +Example: `nextflow run -profile kraken,jenkins` From da2a9047b000947805d2540aadbaba601a02c274 Mon Sep 17 00:00:00 2001 From: Martin Proks Date: Mon, 5 Aug 2019 14:59:36 +0200 Subject: [PATCH 073/138] fix: added kraken to the list (#64) --- README.md | 1 + nfcore_custom.config | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 059aa30..6e68fb4 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Currently documentation is available for the following clusters: * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) +* [KRAKEN](docs/kraken.md) * [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index 2d4c183..9e7d190 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -22,6 +22,7 @@ profiles { czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } + kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } From 332ba0dadb93b0813ac36c9b937c4d8b24fc0b48 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:10:48 +0200 Subject: [PATCH 074/138] fix: update and rearrange munin configuration file and docs --- conf/munin.config | 20 +++++++++++--------- docs/munin.md | 6 ++++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index bca56f7..58e10f7 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -1,20 +1,22 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Big iron cluster profile provided by nf-core/configs.' + config_profile_description = 'Munin profile provided by nf-core/configs.' config_profile_contact = 'Szilveszter Juhos (@szilva)' config_profile_url = '' } process { executor = 'local' + maxForks = 48 } +// To use docker, use nextflow run -profile munin,singularity singularity { enabled = true autoMounts = true } -// To use docker instead of singularity, use nextflow run -profile munin,docker +// To use docker, use nextflow run -profile munin,docker docker { enabled = false mountFlags = 'z' @@ -22,11 +24,11 @@ docker { } params { - saveReference = true - - max_memory = 128.GB - max_cpus = 16 + // general params + max_memory = 754.GB + max_cpus = 48 max_time = 72.h - // illumina iGenomes reference file paths on UPPMAX - igenomes_base = '/data1/references/igenomes/' -} + + // AWS iGenomes reference file paths on munin + igenome_base = '/data1/references/igenomes/' +} \ No newline at end of file diff --git a/docs/munin.md b/docs/munin.md index 17233bb..7f164ce 100644 --- a/docs/munin.md +++ b/docs/munin.md @@ -1,8 +1,10 @@ # nf-core/configs: MUNIN Configuration -All nf-core pipelines have been successfully configured for use on the MUNIN cluster aka big iron. +All nf-core pipelines have been successfully configured for use on the MUNIN cluster. -To use, run the pipeline with `-profile munin`. This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. +To use, run the pipeline with `-profile munin`. +This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. +Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Example: `nextflow run -profile munin` From 12fb2a60c57e85b066d10c18e006759773fe773d Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 7 Aug 2019 11:47:41 +0200 Subject: [PATCH 075/138] Update conf/munin.config Co-Authored-By: Harshil Patel --- conf/munin.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 58e10f7..d837b08 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -10,7 +10,7 @@ process { maxForks = 48 } -// To use docker, use nextflow run -profile munin,singularity +// To use singularity, use nextflow run -profile munin,singularity singularity { enabled = true autoMounts = true @@ -31,4 +31,4 @@ params { // AWS iGenomes reference file paths on munin igenome_base = '/data1/references/igenomes/' -} \ No newline at end of file +} From d6a763214cc8dfa09893a1ea1079b6bad8216f9b Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Wed, 7 Aug 2019 11:47:47 +0200 Subject: [PATCH 076/138] Update conf/munin.config Co-Authored-By: Harshil Patel --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index d837b08..24b5dac 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -29,6 +29,6 @@ params { max_cpus = 48 max_time = 72.h - // AWS iGenomes reference file paths on munin + // Local AWS iGenomes reference file paths on munin igenome_base = '/data1/references/igenomes/' } From 0bd19517b30a1a79b27db1d1163293d38f04ef41 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:55:56 +0200 Subject: [PATCH 077/138] fix: typos --- conf/munin.config | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 58e10f7..b1e4628 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'Munin profile provided by nf-core/configs.' + config_profile_description = 'MUNIN profile provided by nf-core/configs.' config_profile_contact = 'Szilveszter Juhos (@szilva)' config_profile_url = '' } @@ -10,10 +10,9 @@ process { maxForks = 48 } -// To use docker, use nextflow run -profile munin,singularity +// To use singularity, use nextflow run -profile munin,singularity singularity { enabled = true - autoMounts = true } // To use docker, use nextflow run -profile munin,docker @@ -29,6 +28,6 @@ params { max_cpus = 48 max_time = 72.h - // AWS iGenomes reference file paths on munin + // Local AWS iGenomes reference file paths igenome_base = '/data1/references/igenomes/' } \ No newline at end of file From f69524d59a3afdfcb86ddb405f954f27833f3608 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 11:56:10 +0200 Subject: [PATCH 078/138] feat: update docs --- docs/munin.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/munin.md b/docs/munin.md index 7f164ce..167ff9b 100644 --- a/docs/munin.md +++ b/docs/munin.md @@ -2,16 +2,30 @@ All nf-core pipelines have been successfully configured for use on the MUNIN cluster. +## Usage + To use, run the pipeline with `-profile munin`. + This will download and launch the [`munin.config`](../conf/munin.config) which has been pre-configured with a setup suitable for the MUNIN cluster. -Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Example: `nextflow run -profile munin` -## Docker +### Singularity + +This is the default behavior of this configuration profile. + +Using this profile, if no singularity image are available, one will be downloaded from dockerhub, and converted to a Singularity image before execution of the pipeline. + +It is also possible to specify the singularity profile: + +Example: `nextflow run -profile munin,singularity` + +### Docker It is also possible to execute the pipeline using Docker. +Using this profile, if no docker image are available, one will be downloaded from dockerhub before execution of the pipeline. + Example: `nextflow run -profile munin,docker` ## Below are non-mandatory information on iGenomes specific configuration From 4183090379a031e2c5031ab45aa62f1035f88691 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Wed, 7 Aug 2019 12:26:51 +0200 Subject: [PATCH 079/138] fix: reduce cpu and mem usage --- conf/munin.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/munin.config b/conf/munin.config index 9076e38..52062e5 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -7,7 +7,7 @@ params { process { executor = 'local' - maxForks = 48 + maxForks = 46 } // To use singularity, use nextflow run -profile munin,singularity @@ -24,8 +24,8 @@ docker { params { // general params - max_memory = 754.GB - max_cpus = 48 + max_memory = 752.GB + max_cpus = 46 max_time = 72.h // Local AWS iGenomes reference file paths on munin From 486cdfce5f7db8e53372ed92ab53190f0d368a80 Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Wed, 7 Aug 2019 13:19:16 +0200 Subject: [PATCH 080/138] Updating CCGA configs - adding cache dir and mount points --- conf/ccga.config | 1 + conf/ccga_dx.config | 1 + 2 files changed, 2 insertions(+) diff --git a/conf/ccga.config b/conf/ccga.config index b9130e0..6163626 100644 --- a/conf/ccga.config +++ b/conf/ccga.config @@ -14,6 +14,7 @@ params { singularity { enabled = true runOptions = "-B /ifs -B /scratch -B /work_beegfs" + cacheDir = "/ifs/data/nfs_share/ikmb_repository/singularity_cache/" } executor { diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config index 0af95e7..a5c600b 100644 --- a/conf/ccga_dx.config +++ b/conf/ccga_dx.config @@ -13,6 +13,7 @@ params { singularity { enabled = true + runOoptions = "-B /mnt" } executor { From c3c0d088dd01938faa06681af5dd16d747aa08a8 Mon Sep 17 00:00:00 2001 From: Szilveszter Juhos Date: Fri, 16 Aug 2019 11:13:32 +0200 Subject: [PATCH 081/138] typo --- conf/munin.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/munin.config b/conf/munin.config index 52062e5..f53def4 100644 --- a/conf/munin.config +++ b/conf/munin.config @@ -29,5 +29,5 @@ params { max_time = 72.h // Local AWS iGenomes reference file paths on munin - igenome_base = '/data1/references/igenomes/' + igenomes_base = '/data1/references/igenomes/' } From 3b638e9ccdb93ab29c95b80fe5c2c1fb76e6b56d Mon Sep 17 00:00:00 2001 From: marchoeppner Date: Tue, 3 Sep 2019 15:37:15 +0200 Subject: [PATCH 082/138] Fixing typo in ccga_dx config --- conf/ccga_dx.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/ccga_dx.config b/conf/ccga_dx.config index a5c600b..1f92b0d 100644 --- a/conf/ccga_dx.config +++ b/conf/ccga_dx.config @@ -13,7 +13,7 @@ params { singularity { enabled = true - runOoptions = "-B /mnt" + runOptions = "-B /mnt" } executor { From 5e257e7e9a38787e3a440a76d695d28c5183f895 Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 22 Sep 2019 16:45:09 +0200 Subject: [PATCH 083/138] add cbe profile --- README.md | 1 + conf/cbe.config | 32 ++++++++++++++++++++++++++++++++ docs/cbe.md | 18 ++++++++++++++++++ nfcore_custom.config | 1 + 4 files changed, 52 insertions(+) create mode 100755 conf/cbe.config create mode 100644 docs/cbe.md diff --git a/README.md b/README.md index 6e68fb4..9a26324 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ Currently documentation is available for the following clusters: * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) +* [CBE](docs/cbe.md) * [CCGA](docs/ccga.md) * [CCGA_DX](/docs/ccga_dx.md) * [CFC](docs/binac.md) diff --git a/conf/cbe.config b/conf/cbe.config new file mode 100755 index 0000000..5e6aa86 --- /dev/null +++ b/conf/cbe.config @@ -0,0 +1,32 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs' + config_profile_contact = 'Patrick Hüther (@phue)' + config_profile_url = 'http://www.gmi.oeaw.ac.at/' +} + +process { + executor = 'slurm' + module = 'singularity/3.1.0' + queue = 'c' +} + +singularity.enabled = true + +params { + target_qos = 'medium' + params.max_cpus = 36 + params.max_memory = 170.GB + igenomesIgnore = true +} + +if (params.target_qos == 'short') { + params.max_time = 8.h + process.clusterOptions = '--qos short' +} else if (params.target_qos == 'medium') { + params.max_time = 2.d + process.clusterOptions = '--qos medium' +} else { + params.max_time = 14.d + process.clusterOptions = '--qos long' +} diff --git a/docs/cbe.md b/docs/cbe.md new file mode 100644 index 0000000..5cfa5b9 --- /dev/null +++ b/docs/cbe.md @@ -0,0 +1,18 @@ +# nf-core/configs: CBE Configuration + +All nf-core pipelines have been successfully configured for use on the CLIP BATCH ENVIRONMENT (CBE) cluster at the Vienna BioCenter (VBC). + +To use, run the pipeline with `-profile cbe`. This will download and launch the [`cbe.config`](../conf/cbe.config) which has been pre-configured with a setup suitable for the CBE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on CBE. You can do this by issuing the commands below: + +```bash +## Load Nextflow and Singularity environment modules +module purge +module load nextflow/19.04.0 +module load singularity/3.1.0 +``` + +>NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. + +>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/nfcore_custom.config b/nfcore_custom.config index 9e7d190..6e320c4 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -14,6 +14,7 @@ params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/$ profiles { bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } + cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } ccga { includeConfig "${params.custom_config_base}/conf/ccga.config" } ccga_dx { includeConfig "${params.custom_config_base}/conf/ccga_dx.config" } cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } From 014111b925f3d3ebc40eb034e71ee6af06b35813 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:42:11 +0200 Subject: [PATCH 084/138] point travis to local clone Previously travis tests could only succeed if the config was already present in the master branch, but for newly added configs it would always fail. This commit sets custom_config_base to the local clone of the PR branch on the Travis VM where the file is present. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4985029..60bfc96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} From d7e891a5765074259a060c8cdce2ff7b3058bcc7 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:55:43 +0200 Subject: [PATCH 085/138] Revert "point travis to local clone" This reverts commit 014111b925f3d3ebc40eb034e71ee6af06b35813. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 60bfc96..4985029 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} From fb60856e409883d7d1f9cdae808827be1e4d5926 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 23 Sep 2019 13:57:58 +0200 Subject: [PATCH 086/138] point travis to local clone Previously travis tests could only succeed if the config was already present in the master branch, but for newly added configs it would always fail. This commit sets custom_config_base to the local clone of the PR branch on the Travis VM where the file is present. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4985029..9da38dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,4 +27,4 @@ script: tr -s ' ' | \ cut -d " " -f 2 | \ grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf -profile {} + xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} From abb67800816687319bc6febb9d2ffe08412c04e8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 12:48:24 +0200 Subject: [PATCH 087/138] Add Singularity Cache Directory --- conf/cfc.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/cfc.config b/conf/cfc.config index 6fc3b26..0928c00 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,6 +7,7 @@ params { singularity { enabled = true + cacheDir = '/nfsmounts/containers' } process { From 4555dd7d36b7c0b83f0d2bd63166f2cf1fca89af Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 12:51:01 +0200 Subject: [PATCH 088/138] Update conf/cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 0928c00..77a1599 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = '/nfsmounts/containers' + cacheDir = '/nfsmounts/container' } process { From 5fae33669d469fd266b1e87b8aaad5dcffcede43 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:07:28 +0200 Subject: [PATCH 089/138] Update cfc.config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Try this out 👍 --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 77a1599..d1f1112 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = '/nfsmounts/container' + cacheDir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" } process { From 865909ac54d76078fc2c58aed07221acbc14a674 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:14:21 +0200 Subject: [PATCH 090/138] Update cfc.config --- conf/cfc.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index d1f1112..75129b6 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -7,7 +7,7 @@ params { singularity { enabled = true - cacheDir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" + cacheDir = params.singularity_cachedir } process { @@ -22,6 +22,7 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' + singularity_cachedir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" max_memory = 60.GB max_cpus = 20 max_time = 140.h From 1e54d9510dfa6c1be09fb0c21a59c6fece29d846 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:15:49 +0200 Subject: [PATCH 091/138] Update cfc.config --- conf/cfc.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index 75129b6..c852fdd 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -5,9 +5,10 @@ params { config_profile_url = 'http://qbic.uni-tuebingen.de/' } +CACHE_DIR = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" singularity { enabled = true - cacheDir = params.singularity_cachedir + cacheDir = CACHE_DIR } process { @@ -22,7 +23,6 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' - singularity_cachedir = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" max_memory = 60.GB max_cpus = 20 max_time = 140.h From ed8c9d9e7a066e0ab10cced039b8f7949f9ddb7a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 8 Oct 2019 13:20:58 +0200 Subject: [PATCH 092/138] Update cfc.config --- conf/cfc.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/cfc.config b/conf/cfc.config index c852fdd..77a1599 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -5,10 +5,9 @@ params { config_profile_url = 'http://qbic.uni-tuebingen.de/' } -CACHE_DIR = "/nfsmounts/container/nf-core/${workflow.manifest.name}/v${workflow.manifest.version}/" singularity { enabled = true - cacheDir = CACHE_DIR + cacheDir = '/nfsmounts/container' } process { From 855da0cee35054c80bdf43bb012e561a22fac8a1 Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Tue, 8 Oct 2019 14:19:42 +0200 Subject: [PATCH 093/138] Refactors weblog url --- conf/binac.config | 5 +++++ conf/cfc.config | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 0e69d4a..6049f78 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -21,3 +21,8 @@ params { max_cpus = 28 max_time = 48.h } + +weblog{ + enabled = true + url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' +} diff --git a/conf/cfc.config b/conf/cfc.config index 77a1599..d675648 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -17,7 +17,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/workflowservice/workflows' + url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' } params { From dc3faf8a5fe9b63d0c04bbc540a1ce46843f6bbb Mon Sep 17 00:00:00 2001 From: Sven Fillinger Date: Thu, 10 Oct 2019 12:56:39 +0200 Subject: [PATCH 094/138] Sends weblog messages over HTTPS --- conf/binac.config | 2 +- conf/cfc.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index 6049f78..9447ca4 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -24,5 +24,5 @@ params { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' } diff --git a/conf/cfc.config b/conf/cfc.config index d675648..d5b1c3b 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -17,7 +17,7 @@ process { weblog{ enabled = true - url = 'http://services.qbic.uni-tuebingen.de:8080/flowstore/workflows' + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' } params { From 9b3fda36f65bfb1a2f60793df4f7b38f97bb1231 Mon Sep 17 00:00:00 2001 From: phue Date: Thu, 10 Oct 2019 21:22:19 +0200 Subject: [PATCH 095/138] update singularity version singularity/3.1.0 is not available anymore --- conf/cbe.config | 2 +- docs/cbe.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 5e6aa86..c303f92 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -7,7 +7,7 @@ params { process { executor = 'slurm' - module = 'singularity/3.1.0' + module = 'singularity/3.2.1' queue = 'c' } diff --git a/docs/cbe.md b/docs/cbe.md index 5cfa5b9..8a84294 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -10,7 +10,7 @@ Before running the pipeline you will need to load Nextflow and Singularity using ## Load Nextflow and Singularity environment modules module purge module load nextflow/19.04.0 -module load singularity/3.1.0 +module load singularity/3.2.1 ``` >NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. From 1f2786c78f24742f0cb719595efc0e275bc0a1c9 Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 16 Oct 2019 18:15:04 +0200 Subject: [PATCH 096/138] add genouest cluster --- README.md | 1 + conf/genouest.config | 23 +++++++++++++++++++++++ docs/genouest.md | 36 ++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 2 ++ 4 files changed, 62 insertions(+) create mode 100644 conf/genouest.config create mode 100644 docs/genouest.md diff --git a/README.md b/README.md index 9a26324..451c12f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Currently documentation is available for the following clusters: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) diff --git a/conf/genouest.config b/conf/genouest.config new file mode 100644 index 0000000..68d82a6 --- /dev/null +++ b/conf/genouest.config @@ -0,0 +1,23 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The GenOuest cluster profile' + config_profile_contact = 'Anthony Bretaudeau (@abretaud)' + config_profile_url = 'https://www.genouest.org' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /scratch:/scratch -B /local:/local -B /db:/db' +} + +process { + executor = 'slurm' +} + +params { + igenomesIgnore = true + max_memory = 750.GB + max_cpus = 80 + max_time = 336.h +} diff --git a/docs/genouest.md b/docs/genouest.md new file mode 100644 index 0000000..9c37553 --- /dev/null +++ b/docs/genouest.md @@ -0,0 +1,36 @@ +# nf-core/configs: GenOuest Configuration + +All nf-core pipelines have been successfully configured for use on the GenOuest cluster. + +To use, run the pipeline with `-profile genouest`. This will download and launch the [`genouest.config`](../conf/genouest.config) which has been pre-configured with a setup suitable for the GenOuest cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +## Running the workflow on the GenOuest cluster + +Nextflow is installed on the GenOuest cluster. You need to activate it like this: + +```bash +source /local/env/envnextflow-19.07.0.sh +``` + +Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow shouldn't run directly on the submission node but on a compute node. Run nextflow from a compute node: + +```bash +# Login to a compute node +srun --pty bash + +# Load the dependencies if not done before +source /local/env/envnextflow-19.07.0.sh + +# Run a downloaded/git-cloned nextflow workflow from +nextflow run \\ +/path/to/nf-core/workflow \\ +-resume +-profile genouest \\ +--email my-email@example.org \\ +-c my-specific.config +... + +# Or use the nf-core client +nextflow run nf-core/rnaseq ... +``` diff --git a/nfcore_custom.config b/nfcore_custom.config index 6e320c4..06182f5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } @@ -43,6 +44,7 @@ params { // This is a groovy map, not a nextflow parameter set hostnames = [ crick: ['.thecrick.org'], + genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] ] } From 883a761651739b9704b76b386523ee5eb0d072ed Mon Sep 17 00:00:00 2001 From: Anthony Bretaudeau Date: Wed, 16 Oct 2019 18:45:26 +0200 Subject: [PATCH 097/138] add link --- docs/genouest.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/genouest.md b/docs/genouest.md index 9c37553..85341ef 100644 --- a/docs/genouest.md +++ b/docs/genouest.md @@ -6,7 +6,9 @@ To use, run the pipeline with `-profile genouest`. This will download and launch ## Running the workflow on the GenOuest cluster -Nextflow is installed on the GenOuest cluster. You need to activate it like this: +Nextflow is installed on the GenOuest cluster. Some documentation is available on the [GenOuest website](https://www.genouest.org/howto/#nextflow). + +You need to activate it like this: ```bash source /local/env/envnextflow-19.07.0.sh From 5e54912ada55fa05cdca645584a185ed465c6389 Mon Sep 17 00:00:00 2001 From: maxibor Date: Fri, 25 Oct 2019 14:08:02 +0200 Subject: [PATCH 098/138] add queue choice --- conf/shh.config | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 5cb3396..c23d331 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -14,7 +14,14 @@ singularity { process { executor = 'slurm' - queue = 'short' + switch(task.time){ + case {it < 2.h}: + queue = 'short' + case {it > 2.h && it < 48.h}: + queue = 'medium' + case {it > 48.h}: + queue = 'long' + } } executor { From 7a2fe44b43861a89755972048dd944cdacf28484 Mon Sep 17 00:00:00 2001 From: Maxime Borry Date: Fri, 25 Oct 2019 15:50:49 +0200 Subject: [PATCH 099/138] resource based queuing --- conf/shh.config | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index c23d331..14da627 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -14,14 +14,7 @@ singularity { process { executor = 'slurm' - switch(task.time){ - case {it < 2.h}: - queue = 'short' - case {it > 2.h && it < 48.h}: - queue = 'medium' - case {it > 48.h}: - queue = 'long' - } + queue = { task.memory > 756.GB ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } executor { @@ -29,9 +22,9 @@ executor { } params { - max_memory = 256.GB + max_memory = 2.TB max_cpus = 32 - max_time = 2.h + max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" } \ No newline at end of file From 5ec19e3abb25ffa47f09f9123057b31a50108ff9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 27 Oct 2019 13:28:00 +0100 Subject: [PATCH 100/138] Binac SMP / Long / Short Queue Additions --- conf/binac.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/binac.config b/conf/binac.config index 9447ca4..8047bd0 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,6 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { From 123f199b68ff287afacb2e6a3d3fb3078c3ae81f Mon Sep 17 00:00:00 2001 From: phue Date: Sun, 27 Oct 2019 20:44:08 +0100 Subject: [PATCH 101/138] cbe: dynamic qos logic --- conf/cbe.config | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index c303f92..aed9866 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -8,25 +8,15 @@ params { process { executor = 'slurm' module = 'singularity/3.2.1' - queue = 'c' + queue = { task.memory <= 170.GB ? 'c' : 'm' } + clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } } singularity.enabled = true params { - target_qos = 'medium' - params.max_cpus = 36 - params.max_memory = 170.GB - igenomesIgnore = true -} - -if (params.target_qos == 'short') { - params.max_time = 8.h - process.clusterOptions = '--qos short' -} else if (params.target_qos == 'medium') { - params.max_time = 2.d - process.clusterOptions = '--qos medium' -} else { params.max_time = 14.d - process.clusterOptions = '--qos long' -} + params.max_cpus = 36 + params.max_memory = 4.TB + igenomesIgnore = true +} \ No newline at end of file From 18ed48f39839d42e839566f379a1b9e6680b3d3b Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 28 Oct 2019 11:38:24 +0100 Subject: [PATCH 102/138] deprecate igenomesIgnore in favor of igenomes_ignore keep both for now until all the pipelines have been synced --- conf/cbe.config | 3 ++- conf/czbiohub_aws.config | 3 ++- conf/genouest.config | 3 ++- conf/hebbe.config | 3 ++- conf/mendel.config | 3 ++- conf/pasteur.config | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index aed9866..6e47361 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -18,5 +18,6 @@ params { params.max_time = 14.d params.max_cpus = 36 params.max_memory = 4.TB - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated } \ No newline at end of file diff --git a/conf/czbiohub_aws.config b/conf/czbiohub_aws.config index 1dd8da2..98aebe8 100644 --- a/conf/czbiohub_aws.config +++ b/conf/czbiohub_aws.config @@ -55,7 +55,8 @@ params { awsregion = "us-west-2" awsqueue = "nextflow" - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated fc_extra_attributes = 'gene_name' fc_group_features = 'gene_id' diff --git a/conf/genouest.config b/conf/genouest.config index 68d82a6..801b9fb 100644 --- a/conf/genouest.config +++ b/conf/genouest.config @@ -16,7 +16,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated max_memory = 750.GB max_cpus = 80 max_time = 336.h diff --git a/conf/hebbe.config b/conf/hebbe.config index 500ec31..26296d5 100644 --- a/conf/hebbe.config +++ b/conf/hebbe.config @@ -18,7 +18,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated saveReference = true max_memory = 64.GB max_cpus = 20 diff --git a/conf/mendel.config b/conf/mendel.config index e658c87..0fb7e81 100644 --- a/conf/mendel.config +++ b/conf/mendel.config @@ -19,5 +19,6 @@ params { max_cpus = 32 max_memory = 128.GB max_time = 192.h - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated } diff --git a/conf/pasteur.config b/conf/pasteur.config index 1d95131..01d5227 100644 --- a/conf/pasteur.config +++ b/conf/pasteur.config @@ -16,7 +16,8 @@ process { } params { - igenomesIgnore = true + igenomes_ignore = true + igenomesIgnore = true //deprecated max_memory = 256.GB max_cpus = 28 max_time = 24.h From 83fcc0b2baf86d5ef4ca22206c940ce442901008 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:09:30 +0100 Subject: [PATCH 103/138] Adjust max memory --- conf/binac.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index 8047bd0..e2bce86 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,14 +13,14 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 128.GB + max_memory = 1024.GB max_cpus = 28 - max_time = 48.h + max_time = 168.h } weblog{ From 5348b8c609bac0b13003cc464f86ff6973ec1b95 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:12:31 +0100 Subject: [PATCH 104/138] Revert "BINAC SMP / Short / Long Updates" --- conf/binac.config | 5 ++--- docs/pasteur.md | 34 ++++++++++++++++------------------ 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/conf/binac.config b/conf/binac.config index e2bce86..9447ca4 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,14 +13,13 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 1024.GB + max_memory = 128.GB max_cpus = 28 - max_time = 168.h + max_time = 48.h } weblog{ diff --git a/docs/pasteur.md b/docs/pasteur.md index 6e2d825..554761b 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,38 +4,36 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + + ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. - - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) + Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. Nextflow shouldn't run directly on the submission node but on a compute node. The compute nodes don't have access to internet so you need to run it offline. To do that: - 1. Create a virtualenv to install nf-core - - ```bash - module purge - module load Python/3.6.0 - module load java - module load singularity - cd /path/to/nf-core/workflows - virtualenv .venv -p python3 - . .venv/bin/activate - ``` - +```bash +module purge +module load Python/3.6.0 +module load java +module load singularity +cd /path/to/nf-core/workflows +virtualenv .venv -p python3 +. .venv/bin/activate +``` 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) -4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/configs#offline-usage) -5. Run nextflow on a compute node: - +4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) +5. Run nextflow on a compute node: ```bash # create a terminal -tmux +tmux # Get a compute node salloc @@ -48,7 +46,7 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ From 494aa8b53e9830a6f2d332e0d3ee40447a6dda8d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:15:54 +0100 Subject: [PATCH 105/138] Mini fix --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index e2bce86..cba2ab6 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,7 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} } params { From 41e607e7f98c67cb181ceffa625002473997e0c2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:17:39 +0100 Subject: [PATCH 106/138] Fix expression --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index cba2ab6..748c9e6 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -13,7 +13,7 @@ process { beforeScript = 'module load devel/singularity/3.0.3' executor = 'pbs' queue = 'short' - process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'long'} + process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'short' : 'long'} } params { From e557ced1844e6bb6ce74e8f2b038abe2b7024819 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 28 Oct 2019 13:21:41 +0100 Subject: [PATCH 107/138] Update max memory --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index 748c9e6..f669793 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -18,7 +18,7 @@ process { params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 1024.GB + max_memory = 1000.GB max_cpus = 28 max_time = 168.h } From 5a9dbe5d7c5472f3001f5c860ce498b97b073ce4 Mon Sep 17 00:00:00 2001 From: phue Date: Mon, 28 Oct 2019 14:11:34 +0100 Subject: [PATCH 108/138] MENDEL is no more the cluster is no longer operational, was replaced by CBE --- README.md | 1 - conf/mendel.config | 24 ------------------------ docs/mendel.md | 18 ------------------ nfcore_custom.config | 1 - 4 files changed, 44 deletions(-) delete mode 100644 conf/mendel.config delete mode 100644 docs/mendel.md diff --git a/README.md b/README.md index 451c12f..20fafd7 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,6 @@ Currently documentation is available for the following clusters: * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) * [KRAKEN](docs/kraken.md) -* [MENDEL](docs/mendel.md) * [MUNIN](docs/munin.md) * [PASTEUR](docs/pasteur.md) * [PHOENIX](docs/phoenix.md) diff --git a/conf/mendel.config b/conf/mendel.config deleted file mode 100644 index 0fb7e81..0000000 --- a/conf/mendel.config +++ /dev/null @@ -1,24 +0,0 @@ -//Profile config names for nf-core/configs -params { - config_profile_description = 'GMI MENDEL cluster profile provided by nf-core/configs' - config_profile_contact = 'Patrick Hüther (@phue)' - config_profile_url = 'http://www.gmi.oeaw.ac.at/' -} - -manifest { - nextflowVersion = '>=19.01.0' -} - -process { - beforeScript = 'module load Miniconda3/4.6.7' - executor = 'pbspro' - clusterOptions = { "-P $params.project" } -} - -params { - max_cpus = 32 - max_memory = 128.GB - max_time = 192.h - igenomes_ignore = true - igenomesIgnore = true //deprecated -} diff --git a/docs/mendel.md b/docs/mendel.md deleted file mode 100644 index ae0ad4c..0000000 --- a/docs/mendel.md +++ /dev/null @@ -1,18 +0,0 @@ -# nf-core/configs: MENDEL Configuration - -All nf-core pipelines have been successfully configured for use on the MENDEL CLUSTER at the Gregor Mendel Institute (GMI). - -To use, run the pipeline with `-profile conda,mendel`. This will download and launch the [`mendel.config`](../conf/mendel.config) which has been pre-configured with a setup suitable for the MENDEL cluster. A Conda environment will be created automatically and software dependencies will be resolved via [bioconda](https://bioconda.github.io/). - -Before running the pipeline you will need to load Conda using the environment module system on MENDEL. You can do this by issuing the commands below: - -```bash -## Load Nextflow and Conda environment modules -module purge -module load Nextflow -module load Miniconda/4.6.7 -``` - ->NB: You will need an account to use the HPC cluster in order to run the pipeline. If in doubt contact the HPC team. - ->NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact the HPC team. diff --git a/nfcore_custom.config b/nfcore_custom.config index 06182f5..12a0c1e 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -25,7 +25,6 @@ profiles { gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } kraken { includeConfig "${params.custom_config_base}/conf/kraken.config" } - mendel { includeConfig "${params.custom_config_base}/conf/mendel.config" } munin { includeConfig "${params.custom_config_base}/conf/munin.config" } pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } From 47a7d57db4c7d2acec49f468d69eb340286d1d4c Mon Sep 17 00:00:00 2001 From: maxibor Date: Mon, 28 Oct 2019 16:09:00 +0100 Subject: [PATCH 109/138] update shh doc --- docs/shh.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/shh.md b/docs/shh.md index 67cf3b9..aff3f21 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -10,7 +10,8 @@ To use, run the pipeline with `-profile shh`. This will download and launch the however this will likely change to a read-only directory in the future that will be managed by IT. -Note that **the configuration file is currently optimised for `nf-core/eager`**. It will submit to the short queue but with a walltime of 2 hours. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. +Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. From 99f4d9b9bd1bdd7465e9453c0ff209654a0ed8a0 Mon Sep 17 00:00:00 2001 From: arontommi Date: Wed, 6 Nov 2019 13:10:09 +0100 Subject: [PATCH 110/138] documentation on how to fix bianca cluster configuration problem --- docs/uppmax.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/uppmax.md b/docs/uppmax.md index 07a7f14..d5bba37 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -26,6 +26,12 @@ If running on Bianca, you will have no internet connection and these configs wil Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. +Please note that Bianca only allocates 7 GB memory per core so max memory needs to be configured: + +```bash +--max_memory "112GB" +``` + ## Getting more memory If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: From 16a0ba9fd58db31ecd352140d45d70356250322e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 8 Nov 2019 10:06:23 +0100 Subject: [PATCH 111/138] Update cfc.config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index d5b1c3b..355f106 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -11,7 +11,7 @@ singularity { } process { - beforeScript = 'module load qbic/singularity_slurm/3.0.3' + beforeScript = 'module load devel/singularity/3.4.2' executor = 'slurm' } From 5e7d13df89b45932cdc054fd1123d57c49f12dc1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 8 Nov 2019 10:59:34 +0100 Subject: [PATCH 112/138] Update uppmax.md --- docs/uppmax.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/uppmax.md b/docs/uppmax.md index d5bba37..61a1832 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -26,7 +26,7 @@ If running on Bianca, you will have no internet connection and these configs wil Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. -Please note that Bianca only allocates 7 GB memory per core so max memory needs to be configured: +Note that Bianca only allocates 7 GB memory per core so the max memory needs to be limited: ```bash --max_memory "112GB" From d2603a134ed9e1922ad18db8fe747140ae8d8391 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 12:26:19 +0100 Subject: [PATCH 113/138] Add in denbi cloud configs --- conf/denbi_qbic.config | 26 ++++++++++++++++++++++++++ docs/denbi_qbic.md | 8 ++++++++ 2 files changed, 34 insertions(+) create mode 100644 conf/denbi_qbic.config create mode 100644 docs/denbi_qbic.md diff --git a/conf/denbi_qbic.config b/conf/denbi_qbic.config new file mode 100644 index 0000000..d5f1882 --- /dev/null +++ b/conf/denbi_qbic.config @@ -0,0 +1,26 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'de.NBI cluster profile provided by nf-core/configs.' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://cloud.denbi.de/' +} + +singularity { + enabled = true +} + +process { + executor = 'pbs' + queue = 'batch' +} + +params { + max_memory = 512.GB + max_cpus = 28 + max_time = 960.h +} + +weblog{ + enabled = true + url = 'https://services.qbic.uni-tuebingen.de/flowstore/workflows' +} diff --git a/docs/denbi_qbic.md b/docs/denbi_qbic.md new file mode 100644 index 0000000..220aac2 --- /dev/null +++ b/docs/denbi_qbic.md @@ -0,0 +1,8 @@ +# nf-core/configs: de.NBI QBIC Configuration + +All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). + +To use, run the pipeline with `-profile denbi_qbic`. This will download and launch the [`denbi_qbic.config`](../conf/denbi_qbic.config) which has been pre-configured with a setup suitable for the automatically created cluster. Using this profile, a Docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. + +>NB: You will need an account to use de.NBI Cluster in order to run the pipeline. If in doubt contact IT. +>NB: Nextflow will need to submit the jobs via the job scheduler to the cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 3b6c9d0360573b4475dffb0ccd97bc1fc157e624 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 12:28:45 +0100 Subject: [PATCH 114/138] Fix denbi docs --- README.md | 1 + nfcore_custom.config | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 20fafd7..d9cc35f 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,7 @@ Currently documentation is available for the following clusters: * [CRICK](docs/crick.md) * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) +* [DENBI_QBIC](docs/denbi_qbic.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) diff --git a/nfcore_custom.config b/nfcore_custom.config index 12a0c1e..9ec9ade 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } From bc9ecf50acc8f8ca1b8a9e92229f0666223ec2f5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 11 Nov 2019 13:56:17 +0100 Subject: [PATCH 115/138] Update binac.config Upgrade to 3.4.2 --- conf/binac.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/binac.config b/conf/binac.config index d3c9a3c..e34f615 100644 --- a/conf/binac.config +++ b/conf/binac.config @@ -10,7 +10,7 @@ singularity { } process { - beforeScript = 'module load devel/singularity/3.0.3' + beforeScript = 'module load devel/singularity/3.4.2' executor = 'pbs' queue = 'short' process.queue = { task.memory > 128.GB ? 'smp': task.time <= 20.m ? 'tiny' : task.time <= 48.h ? 'short' : task.time <= 168.h ? 'short' : 'long'} From 08a6b748250c26147fc61b827598fef6a5e0a22a Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Thu, 10 Oct 2019 15:33:06 +0200 Subject: [PATCH 116/138] add new profile to be shared: awsbatch --- README.md | 9 +++++---- conf/awsbatch.config | 14 ++++++++++++++ docs/awsbatch.md | 4 ++++ nfcore_custom.config | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 conf/awsbatch.config create mode 100644 docs/awsbatch.md diff --git a/README.md b/README.md index d9cc35f..3ef16e0 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,15 @@ A repository for hosting nextflow config files containing custom parameters requ ## Using an existing config -The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. +The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organization are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organization. ### Configuration and parameters The config files hosted in this repository define a set of parameters which are specific to compute environments at different Institutions but generic enough to be used with all nf-core pipelines. -All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customise the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. +All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. -You should be able to get a good idea as to how other people are customising the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). +You should be able to get a good idea as to how other people are customizing the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). ### Offline usage @@ -80,8 +80,9 @@ You will have to create a [Markdown document](https://www.markdownguide.org/gett See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs) for examples. -Currently documentation is available for the following clusters: +Currently documentation is available for the following systems: +* [AWSBATCH](docs/awsbatch.md) * [BIGPURPLE](docs/bigpurple.md) * [BINAC](docs/binac.md) * [CBE](docs/cbe.md) diff --git a/conf/awsbatch.config b/conf/awsbatch.config new file mode 100644 index 0000000..978da56 --- /dev/null +++ b/conf/awsbatch.config @@ -0,0 +1,14 @@ +//Nextflow config file for running on AWS batch + +params { + config_profile_name = 'AWSBATCH' + config_profile_description = 'AWSBATCH Cloud Profile' + config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_url = 'https://aws.amazon.com/batch/' +} + +aws.region = params.awsregion +process.executor = 'awsbatch' +process.queue = params.awsqueue +executor.awscli = '/home/ec2-user/miniconda/bin/aws' +params.tracedir = './' \ No newline at end of file diff --git a/docs/awsbatch.md b/docs/awsbatch.md new file mode 100644 index 0000000..bec4d78 --- /dev/null +++ b/docs/awsbatch.md @@ -0,0 +1,4 @@ +# nf-core/configs: awsbatch Configuration + +To be used with `awsbatch`. +Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. \ No newline at end of file diff --git a/nfcore_custom.config b/nfcore_custom.config index 9ec9ade..cceb01c 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -12,6 +12,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" profiles { + awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } binac { includeConfig "${params.custom_config_base}/conf/binac.config" } cbe { includeConfig "${params.custom_config_base}/conf/cbe.config" } From cb1c2eacd05476a6a359e1602a95b9bbc91bf92f Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 19 Nov 2019 16:36:51 +0100 Subject: [PATCH 117/138] Z->S --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3ef16e0..b9812b6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ A repository for hosting nextflow config files containing custom parameters requ ## Using an existing config -The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organization are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organization. +The Nextflow [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter can be used with nf-core pipelines in order to load custom config files that you have available locally. However, if you or other people within your organisation are likely to be running nf-core pipelines regularly it may be a good idea to use/create a custom config file that defines some generic settings unique to the computing environment within your organisation. ### Configuration and parameters @@ -24,7 +24,7 @@ The config files hosted in this repository define a set of parameters which are All nf-core pipelines inherit the functionality provided by Nextflow, and as such custom config files can contain parameters/definitions that are available to both. For example, if you have the ability to use [Singularity](https://singularity.lbl.gov/) on your HPC you can add and customize the Nextflow [`singularity`](https://www.nextflow.io/docs/latest/config.html#scope-singularity) scope in your config file. Similarly, you can define a Nextflow [`executor`](https://www.nextflow.io/docs/latest/executor.html) depending on the job submission process available on your cluster. In contrast, the `params` section in your custom config file will typically define parameters that are specific to nf-core pipelines. -You should be able to get a good idea as to how other people are customizing the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). +You should be able to get a good idea as to how other people are customising the execution of their nf-core pipelines by looking at some of the config files in [`nf-core/configs`](https://github.com/nf-core/configs/tree/master/conf). ### Offline usage From b084b9ad3c26676fe62688ede602b5eff00a42f9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:25:49 +0100 Subject: [PATCH 118/138] Adds in GitHub Actions tests --- .github/linting.yml | 20 ++++++++++++++++++++ .github/main.yml | 22 ++++++++++++++++++++++ .travis.yml | 30 ------------------------------ README.md | 14 ++++++++++---- 4 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 .github/linting.yml create mode 100644 .github/main.yml delete mode 100644 .travis.yml diff --git a/.github/linting.yml b/.github/linting.yml new file mode 100644 index 0000000..f6a2458 --- /dev/null +++ b/.github/linting.yml @@ -0,0 +1,20 @@ +name: Markdown linting +# This workflow is triggered on pushes and PRs to the repository. +on: [push, pull_request] + +jobs: + Markdown: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + - name: Run Markdownlint + run: | + markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + + diff --git a/.github/main.yml b/.github/main.yml new file mode 100644 index 0000000..e5f87bf --- /dev/null +++ b/.github/main.yml @@ -0,0 +1,22 @@ +name: nfcore configs tests + +## Check whether users added their config to the matrix +on: [push, pull_request] + paths: + - '.github/main.workflow' + +jobs: + job_1: + name: Run individual profiles + strategy: + matrix: + profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - name: Check ${{ matrix.profile }} profile + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9da38dd..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -sudo: required -language: python -jdk: openjdk8 -services: - - docker -python: - - '3.6' -cache: pip -matrix: - fast_finish: true - -install: - # Install Nextflow - - mkdir /tmp/nextflow && cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests - -env: - - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. - -script: - # Run the pipeline with the test profile and test remote config - - | - grep "{.*includeConfig.*[a-z]*\.config\"" ${TRAVIS_BUILD_DIR}/nfcore_custom.config | \ - tr -s ' ' | \ - cut -d " " -f 2 | \ - grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} diff --git a/README.md b/README.md index b9812b6..53c47e7 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ # ![nf-core/configs](docs/images/nfcore-configs_logo.png) -[![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) +[![Lint Status](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [Using an existing config](#using-an-existing-config) + +* [!nf-core/configs](#nf-coreconfigs) + * [Table of contents](#table-of-contents) + * [Using an existing config](#using-an-existing-config) * [Configuration and parameters](#configuration-and-parameters) * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) + * [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) * [Testing](#testing) * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Help](#help) + * [Help](#help) ## Using an existing config @@ -111,6 +115,8 @@ Currently documentation is available for the following systems: [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. +Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. + Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. From 64a3240a51f6c5c81cc70f71838d0f696833500e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:29:04 +0100 Subject: [PATCH 119/138] Fix linting --- .github/{ => workflows}/linting.yml | 0 .github/{ => workflows}/main.yml | 0 README.md | 25 ++++++++++++------------- 3 files changed, 12 insertions(+), 13 deletions(-) rename .github/{ => workflows}/linting.yml (100%) rename .github/{ => workflows}/main.yml (100%) diff --git a/.github/linting.yml b/.github/workflows/linting.yml similarity index 100% rename from .github/linting.yml rename to .github/workflows/linting.yml diff --git a/.github/main.yml b/.github/workflows/main.yml similarity index 100% rename from .github/main.yml rename to .github/workflows/main.yml diff --git a/README.md b/README.md index 53c47e7..c581c12 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ -# ![nf-core/configs](docs/images/nfcore-configs_logo.png) +[![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) -[![Lint Status](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg) +[![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [!nf-core/configs](#nf-coreconfigs) - * [Table of contents](#table-of-contents) - * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) - * [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) - * [Help](#help) +* [Table of contents](#table-of-contents) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) +* [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Help](#help) ## Using an existing config From 2d169174732d43545c264780db668030d13a30d2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:34:55 +0100 Subject: [PATCH 120/138] Fix linting --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5f87bf..22a3655 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,12 +1,14 @@ name: nfcore configs tests ## Check whether users added their config to the matrix -on: [push, pull_request] - paths: - - '.github/main.workflow' +on: + push: + paths: + - '.github/main.yml' jobs: - job_1: + profile_test: + runs-on: ubuntu-latest name: Run individual profiles strategy: matrix: @@ -14,9 +16,8 @@ jobs: steps: - uses: actions/checkout@v1 - name: Install Nextflow - run: | + run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check ${{ matrix.profile }} profile - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} - + - name: "Check profile" + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file From aeb2fb6dc0a14ac16dd93733f86d3ede2cfd7ddf Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:48:17 +0100 Subject: [PATCH 121/138] Fix all the markdownlinting errors --- .github/markdownlint.yml | 5 ++++ README.md | 23 +++++++++--------- docs/awsbatch.md | 2 +- docs/bigpurple.md | 11 +++++---- docs/binac.md | 2 -- docs/cbe.md | 1 - docs/ccga.md | 8 +++---- docs/ccga_dx.md | 5 ++-- docs/cfc.md | 2 -- docs/crick.md | 1 - docs/czbiohub.md | 52 +++++++++++++++++++--------------------- docs/denbi_qbic.md | 2 +- docs/pasteur.md | 40 +++++++++++++++---------------- docs/prince.md | 8 +++---- docs/shh.md | 1 - docs/template.md | 4 +--- docs/uppmax.md | 5 ++++ docs/uzh.md | 2 -- 18 files changed, 85 insertions(+), 89 deletions(-) create mode 100644 .github/markdownlint.yml diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml new file mode 100644 index 0000000..96b12a7 --- /dev/null +++ b/.github/markdownlint.yml @@ -0,0 +1,5 @@ +# Markdownlint configuration file +default: true, +line-length: false +no-duplicate-header: + siblings_only: true diff --git a/README.md b/README.md index c581c12..f060b03 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) +# [![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) [![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) @@ -6,16 +6,17 @@ A repository for hosting nextflow config files containing custom parameters requ ## Table of contents -* [Table of contents](#table-of-contents) -* [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) -* [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Help](#help) +* [![nf-core/configs](https://github.com/nf-core/configs)](#nf-coreconfigshttpsgithubcomnf-coreconfigs) + * [Table of contents](#table-of-contents) + * [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) + * [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) + * [Help](#help) ## Using an existing config diff --git a/docs/awsbatch.md b/docs/awsbatch.md index bec4d78..94eb7ec 100644 --- a/docs/awsbatch.md +++ b/docs/awsbatch.md @@ -1,4 +1,4 @@ # nf-core/configs: awsbatch Configuration To be used with `awsbatch`. -Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. \ No newline at end of file +Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. diff --git a/docs/bigpurple.md b/docs/bigpurple.md index 19b0352..3673820 100644 --- a/docs/bigpurple.md +++ b/docs/bigpurple.md @@ -1,24 +1,25 @@ # nf-core/configs: BigPurple Configuration + ## nf-core pipelines that use this repo + All nf-core pipelines that use this config repo (which is most), can be run on BigPurple. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. Once in the interactive session: -``` +```bash module load singularity/3.1 module load squashfs-tools/4.3 ``` Now, run the pipeline of your choice with `-profile bigpurple`. This will download and launch the bigpurple.config which has been pre-configured with a setup suitable for the BigPurple cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. -An example commandline: +An example commandline: `nextflow run nf-core/ -profile bigpurple ` ## nf-core pipelines that do not use this repo -If the pipeline has not yet been configured to use this config, then you will have to do it manually. + +If the pipeline has not yet been configured to use this config, then you will have to do it manually. git clone this repo, copy the `bigpurple.config` from the conf folder and then you can invoke the pipeline like this: `nextflow run nf-core/ -c bigpurple.config ` - >NB: You will need an account to use the HPC cluster BigPurple in order to run the pipeline. If in doubt contact MCIT. - >NB: You will need to install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason there is no module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update yourself: `nextflow self-update` diff --git a/docs/binac.md b/docs/binac.md index c6c3819..ce91b91 100644 --- a/docs/binac.md +++ b/docs/binac.md @@ -13,7 +13,5 @@ module load devel/java_jdk/1.8.0u112 module load devel/singularity/3.0.1 ``` - >NB: You will need an account to use the HPC cluster BINAC in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/cbe.md b/docs/cbe.md index 8a84294..5b17a15 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -14,5 +14,4 @@ module load singularity/3.2.1 ``` >NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/ccga.md b/docs/ccga.md index 1157cb3..798df29 100644 --- a/docs/ccga.md +++ b/docs/ccga.md @@ -1,6 +1,6 @@ # nf-core/configs: CCGA Configuration -Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. +Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. @@ -9,9 +9,9 @@ Before running the pipeline you will need to load Nextflow and Singularity using ```bash ## Load Nextflow and Singularity environment modules module purge -module load IKMB -module load Java/1.8.0 -module load Nextflow +module load IKMB +module load Java/1.8.0 +module load Nextflow module load singularity3.1.0 ``` diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md index 0cea81e..162661d 100644 --- a/docs/ccga_dx.md +++ b/docs/ccga_dx.md @@ -1,9 +1,8 @@ # nf-core/configs: CCGA DX Configuration -Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. +Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga_dx.config`](../conf/ccga_dx.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -Before running the pipeline you will need to have Nextflow installed. - +Before running the pipeline you will need to have Nextflow installed. >NB: Access to the CCGA DX cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). diff --git a/docs/cfc.md b/docs/cfc.md index 5cf87d1..a7f6beb 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -13,7 +13,5 @@ module load devel/java_jdk/1.8.0u121 module load qbic/singularity_slurm/3.0.3 ``` - >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/crick.md b/docs/crick.md index 40f5e33..7d76e30 100644 --- a/docs/crick.md +++ b/docs/crick.md @@ -20,5 +20,4 @@ Alternatively, if you are running the pipeline regularly for genomes that arent All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. >NB: You will need an account to use the HPC cluster on CAMP in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/czbiohub.md b/docs/czbiohub.md index 0120283..4826670 100644 --- a/docs/czbiohub.md +++ b/docs/czbiohub.md @@ -14,7 +14,7 @@ The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure [tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop. Start a new tmux session with `tmux new` and we'll name this session `nextflow`. -``` +```bash tmux new -n nextflow ``` @@ -22,89 +22,87 @@ Now you can run pipelines with abandon! ### 2. Make a GitHub repo for your workflows (optional :) - To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. -``` +```bash rnaseq: - nextflow run -profile czbiohub_aws nf-core/rnaseq \ +nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ --outdir s3://olgabot-maca/nextflow-test/ +``` -human_mouse_zebrafish: - nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ - --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv +Human_Mouse_Zebrafish: +```bash +nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ +--samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv +``` -merkin2012_aws: - nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ - -r olgabot/support-csv-directory-or-sra \ - -profile aws +Merkin2012_AWS: + +```bash +nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ +-r olgabot/support-csv-directory-or-sra \-profile aws ``` In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: -``` +```bash make rnaseq ``` If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: -``` +```bash make human_mouse_zebrafish ``` Makefiles are a very useful way of storing longer commands with short mnemonic words. - Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: -``` +```bash git clone https://github.com/czbiohub/kh-workflows ``` Now both create and edit a `Makefile`: -``` +```bash cd nano Makefile ``` Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. - -``` +```bash git add Makefile git commit -m "Added makefile" git push origin master ``` - -### 3. Run your workflow!! - +### 3. Run your workflow Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI -``` +```bash nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ --outdir s3://olgabot-maca/nextflow-test/ ``` -### 4. If you lose connection, how do you restart the jobs? +### 4. If you lose connection, how do you restart the jobs If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to AWS and may need to restart the jobs. To reattach, use the command `tmux attach` and you should see your Nextflow output! To get the named session, use: -``` +```bash tmux attach -n nextflow ``` To restart the jobs from where you left off, add the `-resume` flag to your `nextflow` command: - -``` +```bash nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ @@ -122,7 +120,5 @@ You can do this by simply using the `--genome ` parameter. For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. - >NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/denbi_qbic.md b/docs/denbi_qbic.md index 220aac2..0b759bd 100644 --- a/docs/denbi_qbic.md +++ b/docs/denbi_qbic.md @@ -1,6 +1,6 @@ # nf-core/configs: de.NBI QBIC Configuration -All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). +All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). To use, run the pipeline with `-profile denbi_qbic`. This will download and launch the [`denbi_qbic.config`](../conf/denbi_qbic.config) which has been pre-configured with a setup suitable for the automatically created cluster. Using this profile, a Docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. diff --git a/docs/pasteur.md b/docs/pasteur.md index 554761b..b7d37fa 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,40 +4,40 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - - ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. + - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) - -Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow manages each process as a separate job that is submitted to the cluster by using the `sbatch` command. Nextflow shouldn't run directly on the submission node but on a compute node. -The compute nodes don't have access to internet so you need to run it offline. +The compute nodes don't have access to internet so you need to run it offline. To do that: + 1. Create a virtualenv to install nf-core -```bash -module purge -module load Python/3.6.0 -module load java -module load singularity -cd /path/to/nf-core/workflows -virtualenv .venv -p python3 -. .venv/bin/activate -``` + + ```bash + module purge + module load Python/3.6.0 + module load java + module load singularity + cd /path/to/nf-core/workflows + virtualenv .venv -p python3 + . .venv/bin/activate + ``` + 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) 4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) -5. Run nextflow on a compute node: +5. Run nextflow on a compute node: + ```bash # create a terminal -tmux - +tmux # Get a compute node salloc - # Load the dependencies if not done before module purge module load java @@ -46,11 +46,11 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ --custom_config_base /path/to/configs/from/step/4/ \\ -c my-specific.config -... + ``` diff --git a/docs/prince.md b/docs/prince.md index 12f5e65..d339541 100644 --- a/docs/prince.md +++ b/docs/prince.md @@ -1,5 +1,7 @@ # nf-core/configs: Prince Configuration + ## nf-core pipelines that use this repo + All nf-core pipelines that use this config repo (which is most), can be run on prince. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. Now, run the pipeline of your choice with `-profile prince`. This will download and launch the prince.config which has been pre-configured with a setup suitable for the prince cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. This step **takes time**!! @@ -8,13 +10,11 @@ An example commandline: `nextflow run nf-core/ -profile prince ` ## nf-core pipelines that do not use this repo + If the pipeline has not yet been configured to use this config, then you will have to do it manually. git clone this repo, copy the `prince.config` from the conf folder and then you can invoke the pipeline like this: `nextflow run nf-core/ -c prince.config ` - >NB: You will need an account to use the HPC cluster Prince in order to run the pipeline. If in doubt contact the HPC admins. - ->NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update` - +>NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update`. diff --git a/docs/shh.md b/docs/shh.md index aff3f21..78abed0 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -14,5 +14,4 @@ This configuration will automatically choose the correct SLURM queue (`short`,`m Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. diff --git a/docs/template.md b/docs/template.md index 18f8e4a..2e915f7 100644 --- a/docs/template.md +++ b/docs/template.md @@ -4,7 +4,7 @@ All nf-core pipelines have been successfully configured for use on the PROFILE C To use, run the pipeline with `-profile PROFILENAME`. This will download and launch the [`profile.config`](../conf/profile.config) which has been pre-configured with a setup suitable for the PROFILE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -## Below are non-mandatory information e.g. on modules to load etc. +## Below are non-mandatory information e.g. on modules to load etc Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on PROFILE CLUSTER. You can do this by issuing the commands below: @@ -20,7 +20,5 @@ module load Singularity/2.6.0 A local copy of the iGenomes resource has been made available on PROFILE CLUSTER so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. - >NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/uppmax.md b/docs/uppmax.md index 61a1832..4c58e9b 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -3,6 +3,7 @@ All nf-core pipelines have been successfully configured for use on the Swedish UPPMAX clusters. ## Using the UPPMAX config profile + To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. In addition to this config profile, you will also need to specify an UPPMAX project id. @@ -18,10 +19,12 @@ This config enables Nextflow to manage the pipeline jobs via the Slurm job sched Just run Nextflow on a login node and it will handle everything else. ## Using iGenomes references + A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Running offline with Bianca + If running on Bianca, you will have no internet connection and these configs will not be loaded. Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. @@ -33,6 +36,7 @@ Note that Bianca only allocates 7 GB memory per core so the max memory needs to ``` ## Getting more memory + If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: ```bash @@ -47,6 +51,7 @@ Note that each job will still start with the same request as normal, but restart All jobs will be submitted to fat nodes using this method, so it's only for use in extreme circumstances. ## Uppmax-devel config + If doing pipeline development work on Uppmax, this profile allows for faster testing. Applied after main UPPMAX config, it overwrites certain parts of the config and submits jobs to the `devcore` queue, which has much faster queue times. diff --git a/docs/uzh.md b/docs/uzh.md index d1902f1..0195eef 100644 --- a/docs/uzh.md +++ b/docs/uzh.md @@ -4,7 +4,5 @@ All nf-core pipelines have been successfully configured for use on the UZH clust To use, run the pipeline with `-profile uzh`. This will download and launch the [`uzh.config`](../conf/uzh.config) which has been pre-configured with a setup suitable for the UZH cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - >NB: You will need an account to use the HPC cluster UZH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 4062c8b4dc8704c2904ab5179e5fcc61c2e660a0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:51:08 +0100 Subject: [PATCH 122/138] Fix indentation --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22a3655..7b20832 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,10 +14,10 @@ jobs: matrix: profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] steps: - - uses: actions/checkout@v1 - - name: Install Nextflow - run: | + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + - name: "Check profile" + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file From 5a1dbf2dc795cf92c2f98cd49816ad1d4a795f0b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:52:39 +0100 Subject: [PATCH 123/138] Rename configs tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b20832..341bcf6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: nfcore configs tests +name: Configs tests ## Check whether users added their config to the matrix on: From 3d7f5436b1dc6a70a4163bcba5c4f77fcdce12fa Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:54:17 +0100 Subject: [PATCH 124/138] Its in workflows --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 341bcf6..f53d572 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ name: Configs tests on: push: paths: - - '.github/main.yml' + - '.github/workflows/main.yml' jobs: profile_test: From b60fa8109ebe9a4742c4c74d86ce30ec77b44381 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:57:10 +0100 Subject: [PATCH 125/138] Shift some configs to params scope --- conf/awsbatch.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 978da56..9eceb13 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -5,10 +5,10 @@ params { config_profile_description = 'AWSBATCH Cloud Profile' config_profile_contact = 'Alexander Peltzer (@apeltzer)' config_profile_url = 'https://aws.amazon.com/batch/' + awsregion = 'eu-west-1' + awsqueue = 'test' + tracedir = './' } -aws.region = params.awsregion process.executor = 'awsbatch' -process.queue = params.awsqueue executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' \ No newline at end of file From 7e3efcac5c0bb23547fdb0f8aecb1f8e77664663 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:59:09 +0100 Subject: [PATCH 126/138] Enjoy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f53d572..2873951 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: name: Run individual profiles strategy: matrix: - profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 75c1fd5cd9c4ee8cce5c1bbbd1e2b436a94f4b65 Mon Sep 17 00:00:00 2001 From: MaxUlysse Date: Tue, 26 Nov 2019 10:43:23 +0100 Subject: [PATCH 127/138] fix tests --- conf/awsbatch.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 978da56..89319c5 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -1,14 +1,16 @@ //Nextflow config file for running on AWS batch params { - config_profile_name = 'AWSBATCH' - config_profile_description = 'AWSBATCH Cloud Profile' + awsqueue = '' + awsregion = '' config_profile_contact = 'Alexander Peltzer (@apeltzer)' + config_profile_description = 'AWSBATCH Cloud Profile' + config_profile_name = 'AWSBATCH' config_profile_url = 'https://aws.amazon.com/batch/' + tracedir = './' } aws.region = params.awsregion process.executor = 'awsbatch' process.queue = params.awsqueue -executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' \ No newline at end of file +executor.awscli = '/home/ec2-user/miniconda/bin/aws' \ No newline at end of file From 1ce8ce0c552ffa45a9bf4c57ae8dd40072b69cf8 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 26 Nov 2019 11:00:24 +0100 Subject: [PATCH 128/138] Apply suggestions from code review --- conf/awsbatch.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 89319c5..cf00a53 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -1,8 +1,8 @@ //Nextflow config file for running on AWS batch params { - awsqueue = '' - awsregion = '' + awsqueue = false + awsregion = 'eu-west-1' config_profile_contact = 'Alexander Peltzer (@apeltzer)' config_profile_description = 'AWSBATCH Cloud Profile' config_profile_name = 'AWSBATCH' @@ -13,4 +13,4 @@ params { aws.region = params.awsregion process.executor = 'awsbatch' process.queue = params.awsqueue -executor.awscli = '/home/ec2-user/miniconda/bin/aws' \ No newline at end of file +executor.awscli = '/home/ec2-user/miniconda/bin/aws' From cb876594a332d0d691bebcceb358a1cd9eabbea8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 13:57:32 +0100 Subject: [PATCH 129/138] Proper user tests now :-) --- .github/workflows/main.yml | 3 ++ bin/cchecker.py | 71 ++++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 7 ++-- 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 bin/cchecker.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2873951..5b5d4e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,5 +19,8 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + - name: Check whether profiles are all tested appropriately + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file diff --git a/bin/cchecker.py b/bin/cchecker.py new file mode 100644 index 0000000..8118eb7 --- /dev/null +++ b/bin/cchecker.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +####################################################################### +####################################################################### +## Created on November 26 to check pipeline configs for nf-core/configs +####################################################################### +####################################################################### + +import os +import sys +import argparse +import re + +############################################ +############################################ +## PARSE ARGUMENTS +############################################ +############################################ + +Description = 'Double check custom config file and github actions file to test all cases' +Epilog = """Example usage: python cchecker.py """ + +argParser = argparse.ArgumentParser(description=Description, epilog=Epilog) +## REQUIRED PARAMETERS +argParser.add_argument('CUSTOM_CONFIG', help="Input nfcore_custom.config.") +argParser.add_argument('GITHUB_CONFIG', help="Input Github Actions YAML") + +args = argParser.parse_args() + +############################################ +############################################ +## MAIN FUNCTION +############################################ +############################################ + +def check_config(Config, Github): + + regex = 'includeConfig*' + ERROR_STR = 'ERROR: Please check config file! Did you really update the profiles?' + + ## CHECK Config First + config_profiles = set() + with open(Config, 'r') as cfg: + for line in cfg: + if re.search(regex, line): + hit = line.split('/')[2].split('.')[0] + config_profiles.add(hit) + + ###Check Github Config now + tests = set() + with open(Github, 'r') as ghfile: + for line in ghfile: + if re.search('profile: ', line): + line = line.replace('\'','').replace('[','').replace(']','') + profiles = line.split(':')[1].split(',') + for p in profiles: + tests.add(p) + + print(len(tests)) + print(len(config_profiles)) + + ###Check if sets are equal + if tests == config_profiles: + sys.exit(0) + else: + #Maybe report what is missing here too + #print("Tests don't seem to test these profiles properly!\n") + # print(config_profiles.difference(tests)) + sys.exit(1) + +check_config(Config=args.CUSTOM_CONFIG,Github=args.GITHUB_CONFIG) diff --git a/nfcore_custom.config b/nfcore_custom.config index cceb01c..3ba8055 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,6 +11,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" +//Please use a new line per includeConfig section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } @@ -21,7 +22,8 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } @@ -34,7 +36,8 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; + includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 9caf9821c9f093f8f73203e54615acf1628b5add Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:04:58 +0100 Subject: [PATCH 130/138] Use strip --- bin/cchecker.py | 15 ++++++--------- nfcore_custom.config | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/cchecker.py b/bin/cchecker.py index 8118eb7..029e6d5 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -44,28 +44,25 @@ def check_config(Config, Github): for line in cfg: if re.search(regex, line): hit = line.split('/')[2].split('.')[0] - config_profiles.add(hit) + config_profiles.add(hit.strip()) ###Check Github Config now tests = set() with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line): - line = line.replace('\'','').replace('[','').replace(']','') + line = line.replace('\'','').replace('[','').replace(']','').replace('\n','') profiles = line.split(':')[1].split(',') for p in profiles: - tests.add(p) - - print(len(tests)) - print(len(config_profiles)) - + tests.add(p.strip()) + ###Check if sets are equal if tests == config_profiles: sys.exit(0) else: #Maybe report what is missing here too - #print("Tests don't seem to test these profiles properly!\n") - # print(config_profiles.difference(tests)) + print("Tests don't seem to test these profiles properly!\n") + print(config_profiles.symmetric_difference(tests)) sys.exit(1) check_config(Config=args.CUSTOM_CONFIG,Github=args.GITHUB_CONFIG) diff --git a/nfcore_custom.config b/nfcore_custom.config index 3ba8055..740fcc5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,7 +11,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" -//Please use a new line per includeConfig section to allow easier linting/parsing. Thank you. +//Please use a new line per include Config section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } From ebb89f2714cd32213e5af77131f5f8b469941c48 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:06:30 +0100 Subject: [PATCH 131/138] Don't check that anymore --- .github/workflows/main.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b5d4e7..d9c2f12 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,5 @@ name: Configs tests -## Check whether users added their config to the matrix -on: - push: - paths: - - '.github/workflows/main.yml' - jobs: profile_test: runs-on: ubuntu-latest @@ -19,7 +13,7 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested appropriately + - name: Check whether profiles are all tested run: | python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" From dea11df453a331397c4da073f14123240b55c4b4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:07:25 +0100 Subject: [PATCH 132/138] Both on PR and on push --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..76b0bf4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: Configs tests +on: [pull_request, push] jobs: profile_test: runs-on: ubuntu-latest From c3c7c74d79de2ef8a401123af4b917d456d2b854 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:11:16 +0100 Subject: [PATCH 133/138] Better code --- .github/workflows/main.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 76b0bf4..e60ec19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,18 @@ name: Configs tests on: [pull_request, push] jobs: + test_all_profiles: + runs-on: ubuntu-latest + name: Check if all profiles are tested + steps: + - uses: actions/checkout@v1 + - name: Check whether profiles are all tested + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest name: Run individual profiles + needs: test_all_profiles strategy: matrix: profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] @@ -14,8 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested - run: | - python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 0afe00ded5de63e3928c4f54c329046326701a73 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:19 +0100 Subject: [PATCH 134/138] Generalize name --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e60ec19..85ebe2a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest - name: Run individual profiles + name: Run ${{ matrix.profile }} profile needs: test_all_profiles strategy: matrix: From 47ac2085f7945e6c019b8e3579d2ab75f316198e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:41 +0100 Subject: [PATCH 135/138] Better names --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 85ebe2a..434504f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: "Check profile" + - name: Check ${{ matrix.profile }} profile run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 2ebd2180edb3543ad404efb16c2793e4d4a2fbf4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:27:22 +0100 Subject: [PATCH 136/138] Add in SCRATCH env --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 434504f..c3101f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,4 +24,6 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Check ${{ matrix.profile }} profile + env: + SCRATCH: '~' run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 37c6714cc75ab25ac4d4835228ca286f4eb93538 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:29:52 +0100 Subject: [PATCH 137/138] Nice prince --- conf/prince.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 1c2ea2c..c665d9f 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -18,10 +18,6 @@ singularity { } process { - beforeScript = """ - module load $singularityModule - module load $squashfsModule - """ - .stripIndent() + beforeScript = "module load $singularityModule $squashfsModule" executor = 'slurm' } From bac91d03faacef910b0d90b1daa1abcc41a2c403 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:38:57 +0100 Subject: [PATCH 138/138] Ignore czi stuff for now --- .github/workflows/main.yml | 2 +- bin/cchecker.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3101f3..3288efd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/bin/cchecker.py b/bin/cchecker.py index 029e6d5..eb995fc 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -48,6 +48,9 @@ def check_config(Config, Github): ###Check Github Config now tests = set() + ###Ignore these profiles + ignore_me = ['czbiohub_aws_highpriority', 'czbiohub_aws'] + tests.update(ignore_me) with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line):