From db51c49b2f04e2a39249a7c36aedddfc2f93f913 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 14:54:25 +0100 Subject: [PATCH 01/46] Add genotoul config --- conf/genotoul.config | 27 ++++++++++++++ docs/genotoul.md | 85 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 conf/genotoul.config create mode 100644 docs/genotoul.md diff --git a/conf/genotoul.config b/conf/genotoul.config new file mode 100644 index 0000000..588028d --- /dev/null +++ b/conf/genotoul.config @@ -0,0 +1,27 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Genotoul cluster profile' + config_profile_contact = 'support.bioinfo.genotoul@inra.fr' + config_profile_url = 'http://bioinfo.genotoul.fr/' +} + +singularity { + // need one image per execution + enabled = true + runOptions = '-B /bank -B /work2 -B /work -B /save -B /home' + +} + +process { + executor = 'slurm' +} + +params { + saveReference = true + igenomesIgnore = true + // Max resources requested by a normal node on genotoul. + max_memory = 120.GB + max_cpus = 48 + max_time = 96.h +} + diff --git a/docs/genotoul.md b/docs/genotoul.md new file mode 100644 index 0000000..026a59c --- /dev/null +++ b/docs/genotoul.md @@ -0,0 +1,85 @@ + + + +# nf-core/configs: Bioinfo Genotoul Configuration + +All nf-core pipelines have been successfully configured for use on the Bioinfo Genotoul cluster at the INRA toulouse. + +To use, run the pipeline with `-profile genotoul`. This will download and +launch the [`genotoul.config`](../conf/genotoul.config) which has been +pre-configured with a setup suitable for the Bioinfo Genotoul cluster. + +Using this profile, a docker image containing all of the required software +will be downloaded, and converted to a Singularity image before execution +of the pipeline. Images are stored for all users in following directory `/usr/local/bioinfo/src/NextflowWorkflows/singularity-img/`. + +## Running the workflow ib the Genologin cluster. + +Before running the pipeline you will need to load Nextflow and +Singularity using the environment module system on Genotoul. You can do +this by issuing the commands below: + + +Once connected on our frontal node : +``` +# Login to a compute node +srun --mem=4G --pty bash +``` + +Setup default nextflow and singularity home directory (to be done only one time): +``` +sh /usr/local/bioinfo/src/NextflowWorkflows/create_nfx_dirs.sh +``` + +Load environment : +``` +module purge +module load bioinfo/nfcore-Nextflow-v19.04.0 +``` + +Try a test workflow (for example the methylseq workflow) : +``` +nextflow run nf-core/methylseq -profile genotoul,test +``` + +Create launch script `nfcore-rnaseq.sh` : +``` +#!/bin/bash +#SBATCH -p workq +#SBATCH -t 1:00:00 #time in hour +#SBATCH --mem=4G +#SBATCH --mail-type=BEGIN,END,FAIL + +module load bioinfo/nfcore-Nextflow-v19.04.0 +nextflow run nf-core/methylseq -profile genotoul,test +``` +Launch on the cluster with sbatch: +``` +sbatch nfcore-rnaseq.sh +``` + +## Mounted directory + +By default, available mount points are : + - /bank + - /home + - /save + - /work + - /work2 + + +To have access to specific other mount point (such as nosave or project) +you can add a config profile file with option `-profile` and wich contain: +``` +singularity.runOptions = '-B /diretory/to/mount' +``` + +## Databanks + +A local copy of several genomes are available in `/bank` directory. See +our [databank page](http://bioinfo.genotoul.fr/index.php/resources-2/databanks/) +to search for your favorite genome. + + +>NB: You will need an account to use the HPC cluster on Genotoul in order +to run the pipeline. If in doubt see http://bioinfo.genotoul.fr/. From 42d968fd2a9274a457dcbc80dea61a0bc5170228 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 16:23:25 +0100 Subject: [PATCH 02/46] add genotoul in nfcore_custom --- nfcore_custom.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 06182f5..017d326 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } @@ -44,6 +45,7 @@ params { // This is a groovy map, not a nextflow parameter set hostnames = [ crick: ['.thecrick.org'], + genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] ] From 9e5b76827fc7e3c6dafc1af8a2b69be068bf0761 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 16:37:00 +0100 Subject: [PATCH 03/46] add genotoul in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b9812b6..cb7e3d4 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Currently documentation is available for the following systems: * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) +* [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) From b084b9ad3c26676fe62688ede602b5eff00a42f9 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:25:49 +0100 Subject: [PATCH 04/46] Adds in GitHub Actions tests --- .github/linting.yml | 20 ++++++++++++++++++++ .github/main.yml | 22 ++++++++++++++++++++++ .travis.yml | 30 ------------------------------ README.md | 14 ++++++++++---- 4 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 .github/linting.yml create mode 100644 .github/main.yml delete mode 100644 .travis.yml diff --git a/.github/linting.yml b/.github/linting.yml new file mode 100644 index 0000000..f6a2458 --- /dev/null +++ b/.github/linting.yml @@ -0,0 +1,20 @@ +name: Markdown linting +# This workflow is triggered on pushes and PRs to the repository. +on: [push, pull_request] + +jobs: + Markdown: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '10' + - name: Install markdownlint + run: | + npm install -g markdownlint-cli + - name: Run Markdownlint + run: | + markdownlint ${GITHUB_WORKSPACE} -c ${GITHUB_WORKSPACE}/.github/markdownlint.yml + + diff --git a/.github/main.yml b/.github/main.yml new file mode 100644 index 0000000..e5f87bf --- /dev/null +++ b/.github/main.yml @@ -0,0 +1,22 @@ +name: nfcore configs tests + +## Check whether users added their config to the matrix +on: [push, pull_request] + paths: + - '.github/main.workflow' + +jobs: + job_1: + name: Run individual profiles + strategy: + matrix: + profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] + steps: + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | + wget -qO- get.nextflow.io | bash + sudo mv nextflow /usr/local/bin/ + - name: Check ${{ matrix.profile }} profile + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9da38dd..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -sudo: required -language: python -jdk: openjdk8 -services: - - docker -python: - - '3.6' -cache: pip -matrix: - fast_finish: true - -install: - # Install Nextflow - - mkdir /tmp/nextflow && cd /tmp/nextflow - - wget -qO- get.nextflow.io | bash - - sudo ln -s /tmp/nextflow/nextflow /usr/local/bin/nextflow - - mkdir -p ${TRAVIS_BUILD_DIR}/tests && cd ${TRAVIS_BUILD_DIR}/tests - -env: - - NXF_VER='18.10.1' SCRATCH='~' # Specify a minimum NF version that should be tested and work. Set SCRATCH for prince.config. - - NXF_VER='' SCRATCH='~' # Plus: get the latest NF version and check, that it works. Set SCRATCH for prince.config. - -script: - # Run the pipeline with the test profile and test remote config - - | - grep "{.*includeConfig.*[a-z]*\.config\"" ${TRAVIS_BUILD_DIR}/nfcore_custom.config | \ - tr -s ' ' | \ - cut -d " " -f 2 | \ - grep -v "czbiohub_aws" | \ - xargs -I {} nextflow run ${TRAVIS_BUILD_DIR}/configtest.nf --custom_config_base=${TRAVIS_BUILD_DIR} -profile {} diff --git a/README.md b/README.md index b9812b6..53c47e7 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ # ![nf-core/configs](docs/images/nfcore-configs_logo.png) -[![Build Status](https://travis-ci.org/nf-core/configs.svg?branch=master)](https://travis-ci.org/nf-core/configs) +[![Lint Status](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [Using an existing config](#using-an-existing-config) + +* [!nf-core/configs](#nf-coreconfigs) + * [Table of contents](#table-of-contents) + * [Using an existing config](#using-an-existing-config) * [Configuration and parameters](#configuration-and-parameters) * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) + * [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) * [Testing](#testing) * [Documentation](#documentation) * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Help](#help) + * [Help](#help) ## Using an existing config @@ -111,6 +115,8 @@ Currently documentation is available for the following systems: [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. +Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. + Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. We will be notified automatically when you have created your pull request, and providing that everything adheres to nf-core guidelines we will endeavour to approve your pull request as soon as possible. From 64a3240a51f6c5c81cc70f71838d0f696833500e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:29:04 +0100 Subject: [PATCH 05/46] Fix linting --- .github/{ => workflows}/linting.yml | 0 .github/{ => workflows}/main.yml | 0 README.md | 25 ++++++++++++------------- 3 files changed, 12 insertions(+), 13 deletions(-) rename .github/{ => workflows}/linting.yml (100%) rename .github/{ => workflows}/main.yml (100%) diff --git a/.github/linting.yml b/.github/workflows/linting.yml similarity index 100% rename from .github/linting.yml rename to .github/workflows/linting.yml diff --git a/.github/main.yml b/.github/workflows/main.yml similarity index 100% rename from .github/main.yml rename to .github/workflows/main.yml diff --git a/README.md b/README.md index 53c47e7..c581c12 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ -# ![nf-core/configs](docs/images/nfcore-configs_logo.png) +[![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) -[![Lint Status](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/config/workflows/nfcore%20configs%20tests/badge.svg) +[![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [!nf-core/configs](#nf-coreconfigs) - * [Table of contents](#table-of-contents) - * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) - * [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) - * [Help](#help) +* [Table of contents](#table-of-contents) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) +* [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Help](#help) ## Using an existing config From 2d169174732d43545c264780db668030d13a30d2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:34:55 +0100 Subject: [PATCH 06/46] Fix linting --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5f87bf..22a3655 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,12 +1,14 @@ name: nfcore configs tests ## Check whether users added their config to the matrix -on: [push, pull_request] - paths: - - '.github/main.workflow' +on: + push: + paths: + - '.github/main.yml' jobs: - job_1: + profile_test: + runs-on: ubuntu-latest name: Run individual profiles strategy: matrix: @@ -14,9 +16,8 @@ jobs: steps: - uses: actions/checkout@v1 - name: Install Nextflow - run: | + run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check ${{ matrix.profile }} profile - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} - + - name: "Check profile" + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file From aeb2fb6dc0a14ac16dd93733f86d3ede2cfd7ddf Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:48:17 +0100 Subject: [PATCH 07/46] Fix all the markdownlinting errors --- .github/markdownlint.yml | 5 ++++ README.md | 23 +++++++++--------- docs/awsbatch.md | 2 +- docs/bigpurple.md | 11 +++++---- docs/binac.md | 2 -- docs/cbe.md | 1 - docs/ccga.md | 8 +++---- docs/ccga_dx.md | 5 ++-- docs/cfc.md | 2 -- docs/crick.md | 1 - docs/czbiohub.md | 52 +++++++++++++++++++--------------------- docs/denbi_qbic.md | 2 +- docs/pasteur.md | 40 +++++++++++++++---------------- docs/prince.md | 8 +++---- docs/shh.md | 1 - docs/template.md | 4 +--- docs/uppmax.md | 5 ++++ docs/uzh.md | 2 -- 18 files changed, 85 insertions(+), 89 deletions(-) create mode 100644 .github/markdownlint.yml diff --git a/.github/markdownlint.yml b/.github/markdownlint.yml new file mode 100644 index 0000000..96b12a7 --- /dev/null +++ b/.github/markdownlint.yml @@ -0,0 +1,5 @@ +# Markdownlint configuration file +default: true, +line-length: false +no-duplicate-header: + siblings_only: true diff --git a/README.md b/README.md index c581c12..f060b03 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) +# [![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) [![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) @@ -6,16 +6,17 @@ A repository for hosting nextflow config files containing custom parameters requ ## Table of contents -* [Table of contents](#table-of-contents) -* [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) -* [Adding a new config](#adding-a-new-config) -* [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) -* [Help](#help) +* [![nf-core/configs](https://github.com/nf-core/configs)](#nf-coreconfigshttpsgithubcomnf-coreconfigs) + * [Table of contents](#table-of-contents) + * [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) + * [Adding a new config](#adding-a-new-config) + * [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) + * [Help](#help) ## Using an existing config diff --git a/docs/awsbatch.md b/docs/awsbatch.md index bec4d78..94eb7ec 100644 --- a/docs/awsbatch.md +++ b/docs/awsbatch.md @@ -1,4 +1,4 @@ # nf-core/configs: awsbatch Configuration To be used with `awsbatch`. -Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. \ No newline at end of file +Custom queue and region can be entered with `params.awsqueue` and `params.region` respectively. diff --git a/docs/bigpurple.md b/docs/bigpurple.md index 19b0352..3673820 100644 --- a/docs/bigpurple.md +++ b/docs/bigpurple.md @@ -1,24 +1,25 @@ # nf-core/configs: BigPurple Configuration + ## nf-core pipelines that use this repo + All nf-core pipelines that use this config repo (which is most), can be run on BigPurple. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. Once in the interactive session: -``` +```bash module load singularity/3.1 module load squashfs-tools/4.3 ``` Now, run the pipeline of your choice with `-profile bigpurple`. This will download and launch the bigpurple.config which has been pre-configured with a setup suitable for the BigPurple cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. -An example commandline: +An example commandline: `nextflow run nf-core/ -profile bigpurple ` ## nf-core pipelines that do not use this repo -If the pipeline has not yet been configured to use this config, then you will have to do it manually. + +If the pipeline has not yet been configured to use this config, then you will have to do it manually. git clone this repo, copy the `bigpurple.config` from the conf folder and then you can invoke the pipeline like this: `nextflow run nf-core/ -c bigpurple.config ` - >NB: You will need an account to use the HPC cluster BigPurple in order to run the pipeline. If in doubt contact MCIT. - >NB: You will need to install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason there is no module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update yourself: `nextflow self-update` diff --git a/docs/binac.md b/docs/binac.md index c6c3819..ce91b91 100644 --- a/docs/binac.md +++ b/docs/binac.md @@ -13,7 +13,5 @@ module load devel/java_jdk/1.8.0u112 module load devel/singularity/3.0.1 ``` - >NB: You will need an account to use the HPC cluster BINAC in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/cbe.md b/docs/cbe.md index 8a84294..5b17a15 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -14,5 +14,4 @@ module load singularity/3.2.1 ``` >NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/ccga.md b/docs/ccga.md index 1157cb3..798df29 100644 --- a/docs/ccga.md +++ b/docs/ccga.md @@ -1,6 +1,6 @@ # nf-core/configs: CCGA Configuration -Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. +Deployment and testing of nf-core pipelines at the CCGA cluster is on-going. To use, run the pipeline with `-profile ccga`. This will download and launch the [`ccga.config`](../conf/ccga.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. @@ -9,9 +9,9 @@ Before running the pipeline you will need to load Nextflow and Singularity using ```bash ## Load Nextflow and Singularity environment modules module purge -module load IKMB -module load Java/1.8.0 -module load Nextflow +module load IKMB +module load Java/1.8.0 +module load Nextflow module load singularity3.1.0 ``` diff --git a/docs/ccga_dx.md b/docs/ccga_dx.md index 0cea81e..162661d 100644 --- a/docs/ccga_dx.md +++ b/docs/ccga_dx.md @@ -1,9 +1,8 @@ # nf-core/configs: CCGA DX Configuration -Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. +Deployment and testing of nf-core pipelines at the CCGA DX cluster is on-going. To use, run the pipeline with `-profile ccga_dx`. This will download and launch the [`ccga_dx.config`](../conf/ccga_dx.config) which has been pre-configured with a setup suitable for the CCGA cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -Before running the pipeline you will need to have Nextflow installed. - +Before running the pipeline you will need to have Nextflow installed. >NB: Access to the CCGA DX cluster is restricted to IKMB/CCGA employes. Please talk to Marc Hoeppner to get access (@marchoeppner). diff --git a/docs/cfc.md b/docs/cfc.md index 5cf87d1..a7f6beb 100644 --- a/docs/cfc.md +++ b/docs/cfc.md @@ -13,7 +13,5 @@ module load devel/java_jdk/1.8.0u121 module load qbic/singularity_slurm/3.0.3 ``` - >NB: You will need an account to use the HPC cluster CFC in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/crick.md b/docs/crick.md index 40f5e33..7d76e30 100644 --- a/docs/crick.md +++ b/docs/crick.md @@ -20,5 +20,4 @@ Alternatively, if you are running the pipeline regularly for genomes that arent All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory anyway. >NB: You will need an account to use the HPC cluster on CAMP in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/czbiohub.md b/docs/czbiohub.md index 0120283..4826670 100644 --- a/docs/czbiohub.md +++ b/docs/czbiohub.md @@ -14,7 +14,7 @@ The pipeline will monitor and submit jobs to AWS Batch on your behalf. To ensure [tmux](https://hackernoon.com/a-gentle-introduction-to-tmux-8d784c404340) is a "Terminal Multiplexer" that allows for commands to continue running even when you have closed your laptop. Start a new tmux session with `tmux new` and we'll name this session `nextflow`. -``` +```bash tmux new -n nextflow ``` @@ -22,89 +22,87 @@ Now you can run pipelines with abandon! ### 2. Make a GitHub repo for your workflows (optional :) - To make sharing your pipelines and commands easy between your teammates, it's best to share code in a GitHub repository. One way is to store the commands in a Makefile ([example](https://github.com/czbiohub/kh-workflows/blob/master/nf-kmer-similarity/Makefile)) which can contain multiple `nextflow run` commands so that you don't need to remember the S3 bucket or output directory for every single one. [Makefiles](https://kbroman.org/minimal_make/) are broadly used in the software community for running many complex commands. Makefiles can have a lot of dependencies and be confusing, so we're only going to write *simple* Makefiles. -``` +```bash rnaseq: - nextflow run -profile czbiohub_aws nf-core/rnaseq \ +nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ --outdir s3://olgabot-maca/nextflow-test/ +``` -human_mouse_zebrafish: - nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ - --samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv +Human_Mouse_Zebrafish: +```bash +nextflow run czbiohub/nf-kmer-similarity -latest -profile aws \ +--samples s3://kmer-hashing/hematopoeisis/smartseq2/human_mouse_zebrafish/samples.csv +``` -merkin2012_aws: - nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ - -r olgabot/support-csv-directory-or-sra \ - -profile aws +Merkin2012_AWS: + +```bash +nextflow run czbiohub/nf-kmer-similarity -latest --sra "SRP016501" \ +-r olgabot/support-csv-directory-or-sra \-profile aws ``` In this example, one would run the `rnaseq` rule and the nextflow command beneath it with: -``` +```bash make rnaseq ``` If one wanted to run a different command, e.g. `human_mouse_zebrafish`, they would specify that command instead. For example: -``` +```bash make human_mouse_zebrafish ``` Makefiles are a very useful way of storing longer commands with short mnemonic words. - Once you [create a new repository](https://github.com/organizations/czbiohub/repositories/new) (best to initialize with a `.gitignore`, license - MIT and `README`), clone that repository to your EC2 instance. For example, if the repository is called `kh-workflows`, this is what the command would look like: -``` +```bash git clone https://github.com/czbiohub/kh-workflows ``` Now both create and edit a `Makefile`: -``` +```bash cd nano Makefile ``` Write your rule with a colon after it, and on the next line must be a **tab**, not spaces. Once you're done, exit the program (the `^` command shown in nano means "Control"), write the file, add it to git, commit it, and push it up to GitHub. - -``` +```bash git add Makefile git commit -m "Added makefile" git push origin master ``` - -### 3. Run your workflow!! - +### 3. Run your workflow Remember to specify `-profile czbiohub_aws` to grab the CZ Biohub-specific AWS configurations, and an `--outdir` with an AWS S3 bucket so you don't run out of space on your small AMI -``` +```bash nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ --outdir s3://olgabot-maca/nextflow-test/ ``` -### 4. If you lose connection, how do you restart the jobs? +### 4. If you lose connection, how do you restart the jobs If you close your laptop, get onto the train, or lose WiFi connection, you may lose connection to AWS and may need to restart the jobs. To reattach, use the command `tmux attach` and you should see your Nextflow output! To get the named session, use: -``` +```bash tmux attach -n nextflow ``` To restart the jobs from where you left off, add the `-resume` flag to your `nextflow` command: - -``` +```bash nextflow run -profile czbiohub_aws nf-core/rnaseq \ --reads 's3://czb-maca/Plate_seq/24_month/180626_A00111_0166_BH5LNVDSXX/fastqs/*{R1,R2}*.fastq.gz' \ --genome GRCm38 \ @@ -122,7 +120,5 @@ You can do this by simply using the `--genome ` parameter. For Human and Mouse, we use [GENCODE](https://www.gencodegenes.org/) gene annotations. This doesn't change how you would specify the genome name, only that the pipelines run with the `czbiohub_aws` profile would be with GENCODE rather than iGenomes. - >NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/denbi_qbic.md b/docs/denbi_qbic.md index 220aac2..0b759bd 100644 --- a/docs/denbi_qbic.md +++ b/docs/denbi_qbic.md @@ -1,6 +1,6 @@ # nf-core/configs: de.NBI QBIC Configuration -All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). +All nf-core pipelines have been successfully configured for use on the de.NBI Cloud cluster. This is a virtual cluster that has been set up using the [virtual cluster setup scripts](https://github.com/MaximilianHanussek/virtual_cluster_local_ips). To use, run the pipeline with `-profile denbi_qbic`. This will download and launch the [`denbi_qbic.config`](../conf/denbi_qbic.config) which has been pre-configured with a setup suitable for the automatically created cluster. Using this profile, a Docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. diff --git a/docs/pasteur.md b/docs/pasteur.md index 554761b..b7d37fa 100644 --- a/docs/pasteur.md +++ b/docs/pasteur.md @@ -4,40 +4,40 @@ All nf-core pipelines have been successfully configured for use on the tars clus To use, run the pipeline with `-profile pasteur`. This will download and launch the [`pasteur.config`](../conf/pasteur.config) which has been pre-configured with a setup suitable for the Pasteur cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - - ## Running the workflow on the Pasteur cluster Nextflow is not installed by default on the Pasteur cluster. + - Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#) - -Nextflow manages each process as a separate job that is submitted to the cluster by using the sbatch command. +Nextflow manages each process as a separate job that is submitted to the cluster by using the `sbatch` command. Nextflow shouldn't run directly on the submission node but on a compute node. -The compute nodes don't have access to internet so you need to run it offline. +The compute nodes don't have access to internet so you need to run it offline. To do that: + 1. Create a virtualenv to install nf-core -```bash -module purge -module load Python/3.6.0 -module load java -module load singularity -cd /path/to/nf-core/workflows -virtualenv .venv -p python3 -. .venv/bin/activate -``` + + ```bash + module purge + module load Python/3.6.0 + module load java + module load singularity + cd /path/to/nf-core/workflows + virtualenv .venv -p python3 + . .venv/bin/activate + ``` + 2. Install nf-core: [here](https://nf-co.re/tools#installation) 3. Get nf-core pipeline and container: [here](https://nf-co.re/tools#downloading-pipelines-for-offline-use) 4. Get the nf-core Pasteur profile: [here](https://github.com/nf-core/rnaseq/blob/master/docs/usage.md#--custom_config_base) -5. Run nextflow on a compute node: +5. Run nextflow on a compute node: + ```bash # create a terminal -tmux - +tmux # Get a compute node salloc - # Load the dependencies if not done before module purge module load java @@ -46,11 +46,11 @@ module load singularity # Run nextflow workflow nextflow run \\ /path/to/pipeline-dir/from/step/3/workflow \\ --resume +-resume -profile pasteur \\ -with-singularity /path/to/pipeline-dir/from/step/3/singularity-images/singularity.img \\ --email my-email@pasteur.fr \\ --custom_config_base /path/to/configs/from/step/4/ \\ -c my-specific.config -... + ``` diff --git a/docs/prince.md b/docs/prince.md index 12f5e65..d339541 100644 --- a/docs/prince.md +++ b/docs/prince.md @@ -1,5 +1,7 @@ # nf-core/configs: Prince Configuration + ## nf-core pipelines that use this repo + All nf-core pipelines that use this config repo (which is most), can be run on prince. **Before** running a pipeline for the first time, go into an interactive slurm session on a compute node (`srun --pty --time=02:00:00 -c 2`), as the docker image for the pipeline will need to be pulled and converted. Now, run the pipeline of your choice with `-profile prince`. This will download and launch the prince.config which has been pre-configured with a setup suitable for the prince cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a singularity image before execution of the pipeline. This step **takes time**!! @@ -8,13 +10,11 @@ An example commandline: `nextflow run nf-core/ -profile prince ` ## nf-core pipelines that do not use this repo + If the pipeline has not yet been configured to use this config, then you will have to do it manually. git clone this repo, copy the `prince.config` from the conf folder and then you can invoke the pipeline like this: `nextflow run nf-core/ -c prince.config ` - >NB: You will need an account to use the HPC cluster Prince in order to run the pipeline. If in doubt contact the HPC admins. - ->NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update` - +>NB: Rather than using the nextflow module, I recommend you install nextflow in your home directory - instructions are on nextflow.io (or ask the writer of this profile). The reason this is better than using the module for nextflow on the cluster, is that the development cycle of nextflow is rapid and it's easy to update your installation yourself: `nextflow self-update`. diff --git a/docs/shh.md b/docs/shh.md index aff3f21..78abed0 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -14,5 +14,4 @@ This configuration will automatically choose the correct SLURM queue (`short`,`m Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. diff --git a/docs/template.md b/docs/template.md index 18f8e4a..2e915f7 100644 --- a/docs/template.md +++ b/docs/template.md @@ -4,7 +4,7 @@ All nf-core pipelines have been successfully configured for use on the PROFILE C To use, run the pipeline with `-profile PROFILENAME`. This will download and launch the [`profile.config`](../conf/profile.config) which has been pre-configured with a setup suitable for the PROFILE cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. -## Below are non-mandatory information e.g. on modules to load etc. +## Below are non-mandatory information e.g. on modules to load etc Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on PROFILE CLUSTER. You can do this by issuing the commands below: @@ -20,7 +20,5 @@ module load Singularity/2.6.0 A local copy of the iGenomes resource has been made available on PROFILE CLUSTER so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. - >NB: You will need an account to use the HPC cluster on PROFILE CLUSTER in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. diff --git a/docs/uppmax.md b/docs/uppmax.md index 61a1832..4c58e9b 100644 --- a/docs/uppmax.md +++ b/docs/uppmax.md @@ -3,6 +3,7 @@ All nf-core pipelines have been successfully configured for use on the Swedish UPPMAX clusters. ## Using the UPPMAX config profile + To use, run the pipeline with `-profile uppmax` (one hyphen). This will download and launch the [`uppmax.config`](../conf/uppmax.config) which has been pre-configured with a setup suitable for the UPPMAX servers. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. In addition to this config profile, you will also need to specify an UPPMAX project id. @@ -18,10 +19,12 @@ This config enables Nextflow to manage the pipeline jobs via the Slurm job sched Just run Nextflow on a login node and it will handle everything else. ## Using iGenomes references + A local copy of the iGenomes resource has been made available on all UPPMAX clusters so you should be able to run the pipeline against any reference available in the `igenomes.config`. You can do this by simply using the `--genome ` parameter. ## Running offline with Bianca + If running on Bianca, you will have no internet connection and these configs will not be loaded. Please use the nf-core helper tool on a different system to download the required pipeline files, and transfer them to bianca. This helper tool bundles the config files in this repo together with the pipeline files, so the profile will still be available. @@ -33,6 +36,7 @@ Note that Bianca only allocates 7 GB memory per core so the max memory needs to ``` ## Getting more memory + If your nf-core pipeline run is running out of memory, you can run on a fat node with more memory using the following nextflow flags: ```bash @@ -47,6 +51,7 @@ Note that each job will still start with the same request as normal, but restart All jobs will be submitted to fat nodes using this method, so it's only for use in extreme circumstances. ## Uppmax-devel config + If doing pipeline development work on Uppmax, this profile allows for faster testing. Applied after main UPPMAX config, it overwrites certain parts of the config and submits jobs to the `devcore` queue, which has much faster queue times. diff --git a/docs/uzh.md b/docs/uzh.md index d1902f1..0195eef 100644 --- a/docs/uzh.md +++ b/docs/uzh.md @@ -4,7 +4,5 @@ All nf-core pipelines have been successfully configured for use on the UZH clust To use, run the pipeline with `-profile uzh`. This will download and launch the [`uzh.config`](../conf/uzh.config) which has been pre-configured with a setup suitable for the UZH cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. - >NB: You will need an account to use the HPC cluster UZH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 4062c8b4dc8704c2904ab5179e5fcc61c2e660a0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:51:08 +0100 Subject: [PATCH 08/46] Fix indentation --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22a3655..7b20832 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,10 +14,10 @@ jobs: matrix: profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] steps: - - uses: actions/checkout@v1 - - name: Install Nextflow - run: | + - uses: actions/checkout@v1 + - name: Install Nextflow + run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + - name: "Check profile" + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file From 5a1dbf2dc795cf92c2f98cd49816ad1d4a795f0b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:52:39 +0100 Subject: [PATCH 09/46] Rename configs tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b20832..341bcf6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: nfcore configs tests +name: Configs tests ## Check whether users added their config to the matrix on: From 3d7f5436b1dc6a70a4163bcba5c4f77fcdce12fa Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:54:17 +0100 Subject: [PATCH 10/46] Its in workflows --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 341bcf6..f53d572 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ name: Configs tests on: push: paths: - - '.github/main.yml' + - '.github/workflows/main.yml' jobs: profile_test: From b60fa8109ebe9a4742c4c74d86ce30ec77b44381 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:57:10 +0100 Subject: [PATCH 11/46] Shift some configs to params scope --- conf/awsbatch.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/awsbatch.config b/conf/awsbatch.config index 978da56..9eceb13 100644 --- a/conf/awsbatch.config +++ b/conf/awsbatch.config @@ -5,10 +5,10 @@ params { config_profile_description = 'AWSBATCH Cloud Profile' config_profile_contact = 'Alexander Peltzer (@apeltzer)' config_profile_url = 'https://aws.amazon.com/batch/' + awsregion = 'eu-west-1' + awsqueue = 'test' + tracedir = './' } -aws.region = params.awsregion process.executor = 'awsbatch' -process.queue = params.awsqueue executor.awscli = '/home/ec2-user/miniconda/bin/aws' -params.tracedir = './' \ No newline at end of file From 7e3efcac5c0bb23547fdb0f8aecb1f8e77664663 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 00:59:09 +0100 Subject: [PATCH 12/46] Enjoy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f53d572..2873951 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: name: Run individual profiles strategy: matrix: - profile: ['awsbatch, bigpurple, binac, cbe, ccga_dx, ccga, cfc, crick, czbiohub_aws_highpriority, czbiohub_aws, denbi_qbic, genouest, gis, hebbe, kraken, munin, pasteur, phoenix, prince, shh, uct_hex, uppmax_devel, uppmax, uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 42d99e700666972ad03dc8fda0b81784cc9d8eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 13:18:34 +0100 Subject: [PATCH 13/46] Update genotoul.md --- docs/genotoul.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 026a59c..47e1916 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -1,6 +1,3 @@ - - - # nf-core/configs: Bioinfo Genotoul Configuration All nf-core pipelines have been successfully configured for use on the Bioinfo Genotoul cluster at the INRA toulouse. From cb876594a332d0d691bebcceb358a1cd9eabbea8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 13:57:32 +0100 Subject: [PATCH 14/46] Proper user tests now :-) --- .github/workflows/main.yml | 3 ++ bin/cchecker.py | 71 ++++++++++++++++++++++++++++++++++++++ nfcore_custom.config | 7 ++-- 3 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 bin/cchecker.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2873951..5b5d4e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,5 +19,8 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ + - name: Check whether profiles are all tested appropriately + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file diff --git a/bin/cchecker.py b/bin/cchecker.py new file mode 100644 index 0000000..8118eb7 --- /dev/null +++ b/bin/cchecker.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +####################################################################### +####################################################################### +## Created on November 26 to check pipeline configs for nf-core/configs +####################################################################### +####################################################################### + +import os +import sys +import argparse +import re + +############################################ +############################################ +## PARSE ARGUMENTS +############################################ +############################################ + +Description = 'Double check custom config file and github actions file to test all cases' +Epilog = """Example usage: python cchecker.py """ + +argParser = argparse.ArgumentParser(description=Description, epilog=Epilog) +## REQUIRED PARAMETERS +argParser.add_argument('CUSTOM_CONFIG', help="Input nfcore_custom.config.") +argParser.add_argument('GITHUB_CONFIG', help="Input Github Actions YAML") + +args = argParser.parse_args() + +############################################ +############################################ +## MAIN FUNCTION +############################################ +############################################ + +def check_config(Config, Github): + + regex = 'includeConfig*' + ERROR_STR = 'ERROR: Please check config file! Did you really update the profiles?' + + ## CHECK Config First + config_profiles = set() + with open(Config, 'r') as cfg: + for line in cfg: + if re.search(regex, line): + hit = line.split('/')[2].split('.')[0] + config_profiles.add(hit) + + ###Check Github Config now + tests = set() + with open(Github, 'r') as ghfile: + for line in ghfile: + if re.search('profile: ', line): + line = line.replace('\'','').replace('[','').replace(']','') + profiles = line.split(':')[1].split(',') + for p in profiles: + tests.add(p) + + print(len(tests)) + print(len(config_profiles)) + + ###Check if sets are equal + if tests == config_profiles: + sys.exit(0) + else: + #Maybe report what is missing here too + #print("Tests don't seem to test these profiles properly!\n") + # print(config_profiles.difference(tests)) + sys.exit(1) + +check_config(Config=args.CUSTOM_CONFIG,Github=args.GITHUB_CONFIG) diff --git a/nfcore_custom.config b/nfcore_custom.config index cceb01c..3ba8055 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,6 +11,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" +//Please use a new line per includeConfig section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } @@ -21,7 +22,8 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } @@ -34,7 +36,8 @@ profiles { shh { includeConfig "${params.custom_config_base}/conf/shh.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } - uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } + uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config"; + includeConfig "${params.custom_config_base}/conf/uppmax_devel.config" } uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" } } From 9caf9821c9f093f8f73203e54615acf1628b5add Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:04:58 +0100 Subject: [PATCH 15/46] Use strip --- bin/cchecker.py | 15 ++++++--------- nfcore_custom.config | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/bin/cchecker.py b/bin/cchecker.py index 8118eb7..029e6d5 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -44,28 +44,25 @@ def check_config(Config, Github): for line in cfg: if re.search(regex, line): hit = line.split('/')[2].split('.')[0] - config_profiles.add(hit) + config_profiles.add(hit.strip()) ###Check Github Config now tests = set() with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line): - line = line.replace('\'','').replace('[','').replace(']','') + line = line.replace('\'','').replace('[','').replace(']','').replace('\n','') profiles = line.split(':')[1].split(',') for p in profiles: - tests.add(p) - - print(len(tests)) - print(len(config_profiles)) - + tests.add(p.strip()) + ###Check if sets are equal if tests == config_profiles: sys.exit(0) else: #Maybe report what is missing here too - #print("Tests don't seem to test these profiles properly!\n") - # print(config_profiles.difference(tests)) + print("Tests don't seem to test these profiles properly!\n") + print(config_profiles.symmetric_difference(tests)) sys.exit(1) check_config(Config=args.CUSTOM_CONFIG,Github=args.GITHUB_CONFIG) diff --git a/nfcore_custom.config b/nfcore_custom.config index 3ba8055..740fcc5 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -11,7 +11,7 @@ params.custom_config_version = 'master' params.custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" -//Please use a new line per includeConfig section to allow easier linting/parsing. Thank you. +//Please use a new line per include Config section to allow easier linting/parsing. Thank you. profiles { awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" } bigpurple { includeConfig "${params.custom_config_base}/conf/bigpurple.config" } From ebb89f2714cd32213e5af77131f5f8b469941c48 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:06:30 +0100 Subject: [PATCH 16/46] Don't check that anymore --- .github/workflows/main.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b5d4e7..d9c2f12 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,5 @@ name: Configs tests -## Check whether users added their config to the matrix -on: - push: - paths: - - '.github/workflows/main.yml' - jobs: profile_test: runs-on: ubuntu-latest @@ -19,7 +13,7 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested appropriately + - name: Check whether profiles are all tested run: | python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" From dea11df453a331397c4da073f14123240b55c4b4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:07:25 +0100 Subject: [PATCH 17/46] Both on PR and on push --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..76b0bf4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: Configs tests +on: [pull_request, push] jobs: profile_test: runs-on: ubuntu-latest From c3c7c74d79de2ef8a401123af4b917d456d2b854 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:11:16 +0100 Subject: [PATCH 18/46] Better code --- .github/workflows/main.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 76b0bf4..e60ec19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,18 @@ name: Configs tests on: [pull_request, push] jobs: + test_all_profiles: + runs-on: ubuntu-latest + name: Check if all profiles are tested + steps: + - uses: actions/checkout@v1 + - name: Check whether profiles are all tested + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest name: Run individual profiles + needs: test_all_profiles strategy: matrix: profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] @@ -14,8 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested - run: | - python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 0afe00ded5de63e3928c4f54c329046326701a73 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:19 +0100 Subject: [PATCH 19/46] Generalize name --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e60ec19..85ebe2a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest - name: Run individual profiles + name: Run ${{ matrix.profile }} profile needs: test_all_profiles strategy: matrix: From 47ac2085f7945e6c019b8e3579d2ab75f316198e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:41 +0100 Subject: [PATCH 20/46] Better names --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 85ebe2a..434504f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: "Check profile" + - name: Check ${{ matrix.profile }} profile run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From bdf4a829b808ec601ebc456669f6a707c5908e6d Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 26 Nov 2019 14:22:16 +0100 Subject: [PATCH 21/46] roll out iGenomes on cbe --- conf/cbe.config | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conf/cbe.config b/conf/cbe.config index 6e47361..3332b1e 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -12,12 +12,14 @@ process { clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } } -singularity.enabled = true +singularity { + enabled = true + cacheDir = '/scratch-cbe/shared/containers' +} params { params.max_time = 14.d params.max_cpus = 36 params.max_memory = 4.TB - igenomes_ignore = true - igenomesIgnore = true //deprecated -} \ No newline at end of file + igenomes_base = '/resources/references/igenomes' +} From 2ebd2180edb3543ad404efb16c2793e4d4a2fbf4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:27:22 +0100 Subject: [PATCH 22/46] Add in SCRATCH env --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 434504f..c3101f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,4 +24,6 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Check ${{ matrix.profile }} profile + env: + SCRATCH: '~' run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 37c6714cc75ab25ac4d4835228ca286f4eb93538 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:29:52 +0100 Subject: [PATCH 23/46] Nice prince --- conf/prince.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 1c2ea2c..c665d9f 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -18,10 +18,6 @@ singularity { } process { - beforeScript = """ - module load $singularityModule - module load $squashfsModule - """ - .stripIndent() + beforeScript = "module load $singularityModule $squashfsModule" executor = 'slurm' } From bac91d03faacef910b0d90b1daa1abcc41a2c403 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:38:57 +0100 Subject: [PATCH 24/46] Ignore czi stuff for now --- .github/workflows/main.yml | 2 +- bin/cchecker.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3101f3..3288efd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/bin/cchecker.py b/bin/cchecker.py index 029e6d5..eb995fc 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -48,6 +48,9 @@ def check_config(Config, Github): ###Check Github Config now tests = set() + ###Ignore these profiles + ignore_me = ['czbiohub_aws_highpriority', 'czbiohub_aws'] + tests.update(ignore_me) with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line): From 42c949db95947490733d5aec4c150760102006ef Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:43:42 +0100 Subject: [PATCH 25/46] correct md to pass linter --- docs/genotoul.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 47e1916..831e90e 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -10,37 +10,41 @@ Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Images are stored for all users in following directory `/usr/local/bioinfo/src/NextflowWorkflows/singularity-img/`. -## Running the workflow ib the Genologin cluster. +## Running the workflow ib the Genologin cluster Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on Genotoul. You can do this by issuing the commands below: - Once connected on our frontal node : -``` + +```bash # Login to a compute node srun --mem=4G --pty bash ``` Setup default nextflow and singularity home directory (to be done only one time): -``` + +```bash sh /usr/local/bioinfo/src/NextflowWorkflows/create_nfx_dirs.sh ``` Load environment : -``` + +```bash module purge module load bioinfo/nfcore-Nextflow-v19.04.0 ``` Try a test workflow (for example the methylseq workflow) : -``` + +```bash nextflow run nf-core/methylseq -profile genotoul,test ``` Create launch script `nfcore-rnaseq.sh` : -``` + +```bash #!/bin/bash #SBATCH -p workq #SBATCH -t 1:00:00 #time in hour @@ -51,7 +55,8 @@ module load bioinfo/nfcore-Nextflow-v19.04.0 nextflow run nf-core/methylseq -profile genotoul,test ``` Launch on the cluster with sbatch: -``` + +```bash sbatch nfcore-rnaseq.sh ``` @@ -64,10 +69,9 @@ By default, available mount points are : - /work - /work2 - To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and wich contain: -``` +```bash singularity.runOptions = '-B /diretory/to/mount' ``` @@ -77,6 +81,5 @@ A local copy of several genomes are available in `/bank` directory. See our [databank page](http://bioinfo.genotoul.fr/index.php/resources-2/databanks/) to search for your favorite genome. - >NB: You will need an account to use the HPC cluster on Genotoul in order -to run the pipeline. If in doubt see http://bioinfo.genotoul.fr/. +to run the pipeline. If in doubt see [http://bioinfo.genotoul.fr/](http://bioinfo.genotoul.fr/). From e95196451fe850201e0b471e2b3cc81d78e210b3 Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:47:23 +0100 Subject: [PATCH 26/46] correct md to pass linter --- docs/genotoul.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 831e90e..5f46f21 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -54,6 +54,7 @@ Create launch script `nfcore-rnaseq.sh` : module load bioinfo/nfcore-Nextflow-v19.04.0 nextflow run nf-core/methylseq -profile genotoul,test ``` + Launch on the cluster with sbatch: ```bash @@ -63,14 +64,15 @@ sbatch nfcore-rnaseq.sh ## Mounted directory By default, available mount points are : - - /bank - - /home - - /save - - /work - - /work2 +- /bank +- /home +- /save +- /work +- /work2 To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and wich contain: + ```bash singularity.runOptions = '-B /diretory/to/mount' ``` From dd4ff452dac7efc00fd04501a460656b1595272f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:49:47 +0100 Subject: [PATCH 27/46] Update conf/genotoul.config Co-Authored-By: Maxime Garcia --- conf/genotoul.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/genotoul.config b/conf/genotoul.config index 588028d..e5a1242 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -18,10 +18,9 @@ process { params { saveReference = true - igenomesIgnore = true + igenomes_ignore = true // Max resources requested by a normal node on genotoul. max_memory = 120.GB max_cpus = 48 max_time = 96.h } - From 611c318569c4d9d4683052ec1d8009c41d05f981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:50:05 +0100 Subject: [PATCH 28/46] Update conf/genotoul.config Co-Authored-By: Maxime Garcia --- conf/genotoul.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/genotoul.config b/conf/genotoul.config index e5a1242..75ad2c4 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -17,7 +17,7 @@ process { } params { - saveReference = true + save_reference = true igenomes_ignore = true // Max resources requested by a normal node on genotoul. max_memory = 120.GB From 288294994e0c9df795cc34c71464ae8a40663e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:52:32 +0100 Subject: [PATCH 29/46] Update docs/genotoul.md ok Co-Authored-By: Maxime Garcia --- docs/genotoul.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 5f46f21..e6b2f13 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -71,7 +71,7 @@ By default, available mount points are : - /work2 To have access to specific other mount point (such as nosave or project) -you can add a config profile file with option `-profile` and wich contain: +you can add a config profile file with option `-profile` and which contain: ```bash singularity.runOptions = '-B /diretory/to/mount' From 3ed206b7a773efdde19e6291e41cd7ecb6e40f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:52:45 +0100 Subject: [PATCH 30/46] Update docs/genotoul.md ok Co-Authored-By: Maxime Garcia --- docs/genotoul.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index e6b2f13..5fe02b7 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -74,7 +74,7 @@ To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and which contain: ```bash -singularity.runOptions = '-B /diretory/to/mount' +singularity.runOptions = '-B /directory/to/mount' ``` ## Databanks From 3d8da62db3c29d0c631f6167112208e3fdd0972a Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:57:17 +0100 Subject: [PATCH 31/46] add genotoul profile name to the build matrix --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..6d3f127 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: name: Run individual profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow @@ -17,4 +17,4 @@ jobs: run: | python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 07b1f17d6a81a145d3c1afde1cf8ebd53b746a4a Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 15:02:06 +0100 Subject: [PATCH 32/46] correct md to pass linter --- docs/genotoul.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 5fe02b7..b86cf66 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -63,12 +63,13 @@ sbatch nfcore-rnaseq.sh ## Mounted directory -By default, available mount points are : -- /bank -- /home -- /save -- /work -- /work2 +By default, available mount points are: + +* /bank +* /home +* /save +* /work +* /work2 To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and which contain: From 0fdcf8c70c86b18c648cbf303b02514a6fa0cda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 15:10:13 +0100 Subject: [PATCH 33/46] Update .github/workflows/main.yml Co-Authored-By: Alexander Peltzer --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e36b79a..a9d41b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 9cb5137ef225888e5e466d2a339c5b78f07db5e0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 15:12:42 +0100 Subject: [PATCH 34/46] Better description --- README.md | 2 +- bin/cchecker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f060b03..58a65d9 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Currently documentation is available for the following systems: [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. -Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. +Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to the alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. If you forget to do this, tests will fail and complain about that. Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. diff --git a/bin/cchecker.py b/bin/cchecker.py index eb995fc..4290170 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -64,7 +64,7 @@ def check_config(Config, Github): sys.exit(0) else: #Maybe report what is missing here too - print("Tests don't seem to test these profiles properly!\n") + print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n") print(config_profiles.symmetric_difference(tests)) sys.exit(1) From ca6ddcadb07d44725c12c81af27f86f326753eed Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:25:35 +0100 Subject: [PATCH 35/46] Matching max CPUs to SDAG (supercruncher8) --- conf/shh.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 14da627..8a7f812 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -23,8 +23,8 @@ executor { params { max_memory = 2.TB - max_cpus = 32 + max_cpus = 128 max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" -} \ No newline at end of file +} From c3d27099ca02436fc3462504dfddfbfd4ceb3961 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:26:05 +0100 Subject: [PATCH 36/46] Updated authors --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 8a7f812..424a0ee 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,7 +1,7 @@ //Profile config names for nf-core/configs params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' - config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' } From 12ec22838c5a23f74940b2912aa98bd3e3a78419 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:29:03 +0100 Subject: [PATCH 37/46] Specified max sources currently at SDAG limits (exceeds cdag) --- docs/shh.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/shh.md b/docs/shh.md index 78abed0..ec268b5 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -11,7 +11,11 @@ To use, run the pipeline with `-profile shh`. This will download and launch the however this will likely change to a read-only directory in the future that will be managed by IT. This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. + Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. + >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. + +>NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. From 89506a653cbe6edbbab5459febfae40ffe9f9cff Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:34:25 +0100 Subject: [PATCH 38/46] Remove empty lines in block quotes for linting --- docs/shh.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/shh.md b/docs/shh.md index ec268b5..c89d550 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -15,7 +15,5 @@ This configuration will automatically choose the correct SLURM queue (`short`,`m Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. - >NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. From da922b80059f7432737aa50f8140910bd9507f23 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 15:45:33 +0100 Subject: [PATCH 39/46] Fix README --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ec810d7..2b18dcc 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ -# [![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) +# [![nf-core/configs](docs/images/nfcore-configs_logo.png "nf-core/configs")](https://github.com/nf-core/configs) -[![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) +[![Lint Status](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [![nf-core/configs](https://github.com/nf-core/configs)](#nf-coreconfigshttpsgithubcomnf-coreconfigs) - * [Table of contents](#table-of-contents) - * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) - * [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) - * [Help](#help) +* [Table of contents](#table-of-contents) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) +* [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Help](#help) ## Using an existing config From b756ce54a488e2f65a63830fb88c6289f98b3426 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 16:09:08 +0100 Subject: [PATCH 40/46] Add genotoul fix --- conf/genotoul.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/genotoul.config b/conf/genotoul.config index 75ad2c4..8ad50b1 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -19,6 +19,7 @@ process { params { save_reference = true igenomes_ignore = true + igenomesIgnore = true //deprecated // Max resources requested by a normal node on genotoul. max_memory = 120.GB max_cpus = 48 From 834f4b417922353b6925c192b03031d3962492d7 Mon Sep 17 00:00:00 2001 From: ggabernet Date: Tue, 26 Nov 2019 17:30:41 +0100 Subject: [PATCH 41/46] updated memory cfc config --- conf/cfc.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cfc.config b/conf/cfc.config index 355f106..f56beea 100644 --- a/conf/cfc.config +++ b/conf/cfc.config @@ -22,7 +22,7 @@ weblog{ params { igenomes_base = '/nfsmounts/igenomes' - max_memory = 60.GB + max_memory = 500.GB max_cpus = 20 max_time = 140.h } From 672f025639ca52a95dce9aee8a23c1bc52f2bad2 Mon Sep 17 00:00:00 2001 From: phue Date: Tue, 26 Nov 2019 18:09:20 +0100 Subject: [PATCH 42/46] cbe: add documentation about iGenomes --- docs/cbe.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/cbe.md b/docs/cbe.md index 5b17a15..8d597a0 100644 --- a/docs/cbe.md +++ b/docs/cbe.md @@ -13,5 +13,7 @@ module load nextflow/19.04.0 module load singularity/3.2.1 ``` +A local copy of the [AWS-iGenomes](https://registry.opendata.aws/aws-igenomes/) resource has been made available on CBE so you should be able to run the pipeline against any reference available in the `igenomes.config` specific to the nf-core pipeline. You can do this by simply using the `--genome ` parameter. + >NB: You will need an account to use the HPC cluster on CBE in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT. From 6f40c134731c64c5ffb33aafc19db7bed2940656 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Tue, 26 Nov 2019 20:20:17 +0100 Subject: [PATCH 43/46] Split shh config for each cluster --- conf/shh_cdag.config | 30 ++++++++++++++++++++++++++++ conf/{shh.config => shh_sdag.config} | 4 ++-- docs/shh.md | 11 ++++------ 3 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 conf/shh_cdag.config rename conf/{shh.config => shh_sdag.config} (71%) diff --git a/conf/shh_cdag.config b/conf/shh_cdag.config new file mode 100644 index 0000000..c0a75c0 --- /dev/null +++ b/conf/shh_cdag.config @@ -0,0 +1,30 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'MPI-SHH CDAG cluster profile provided by nf-core/configs.' + config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' + config_profile_url = 'https://shh.mpg.de' +} + +singularity { + enabled = true + autoMounts = true + runOptions = '-B /run/shm:/run/shm' + cacheDir = "/projects1/singularity_scratch/cache/" +} + +process { + executor = 'slurm' + queue = { task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } +} + +executor { + queueSize = 16 +} + +params { + max_memory = 256.GB + max_cpus = 32 + max_time = 720.h + //Illumina iGenomes reference file path + igenomes_base = "/projects1/public_data/igenomes/" +} diff --git a/conf/shh.config b/conf/shh_sdag.config similarity index 71% rename from conf/shh.config rename to conf/shh_sdag.config index 424a0ee..b865123 100644 --- a/conf/shh.config +++ b/conf/shh_sdag.config @@ -1,6 +1,6 @@ //Profile config names for nf-core/configs params { - config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' + config_profile_description = 'MPI-SHH SDAG cluster profile provided by nf-core/configs.' config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' } @@ -14,7 +14,7 @@ singularity { process { executor = 'slurm' - queue = { task.memory > 756.GB ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } + queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' } } executor { diff --git a/docs/shh.md b/docs/shh.md index c89d550..02b3ca7 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -2,18 +2,15 @@ All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de). -To use, run the pipeline with `-profile shh`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: +To use, run the pipeline either with `-profile shh_sdag` or `-profile ssh_cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here: ```bash /projects1/singularity_scratch/cache/ ``` -however this will likely change to a read-only directory in the future that will be managed by IT. +however this will likely change to a read-only directory in the future that will be managed by the IT team. -This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. +This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `ssh_sdag` additionally allows for submission of jobs to the `supercruncher` when a job's requested memory exceeds 756GB. -Please note that there is no `supercruncher` queue on CDAG. - ->NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. +>NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team. >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. ->NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. From e6bf4709629b223bd9f2acd4b4ce3385e458cf28 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Tue, 26 Nov 2019 20:23:04 +0100 Subject: [PATCH 44/46] Updated github actions test --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3288efd..a4a5cef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh_sdag', 'ssh_cdag', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From c89ca84244900ad54562fb9a86297c24e706bbba Mon Sep 17 00:00:00 2001 From: jfy133 Date: Tue, 26 Nov 2019 20:24:23 +0100 Subject: [PATCH 45/46] Updated github actions test - typo fix --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4a5cef..073f44b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh_sdag', 'ssh_cdag', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh_sdag', 'shh_cdag', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From e1c8c86d7030678a7b5f26ee8ac559d2e77babb2 Mon Sep 17 00:00:00 2001 From: jfy133 Date: Tue, 26 Nov 2019 20:26:59 +0100 Subject: [PATCH 46/46] Updated core config --- nfcore_custom.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nfcore_custom.config b/nfcore_custom.config index 740fcc5..ebd778b 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -33,7 +33,8 @@ profiles { pasteur { includeConfig "${params.custom_config_base}/conf/pasteur.config" } phoenix { includeConfig "${params.custom_config_base}/conf/phoenix.config" } prince { includeConfig "${params.custom_config_base}/conf/prince.config" } - shh { includeConfig "${params.custom_config_base}/conf/shh.config" } + shh_sdag { includeConfig "${params.custom_config_base}/conf/shh_sdag.config" } + shh_cdag { includeConfig "${params.custom_config_base}/conf/shh_cdag.config" } uct_hex { includeConfig "${params.custom_config_base}/conf/uct_hex.config" } uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" } uppmax_devel { includeConfig "${params.custom_config_base}/conf/uppmax.config";