From db51c49b2f04e2a39249a7c36aedddfc2f93f913 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 14:54:25 +0100 Subject: [PATCH 01/27] Add genotoul config --- conf/genotoul.config | 27 ++++++++++++++ docs/genotoul.md | 85 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 conf/genotoul.config create mode 100644 docs/genotoul.md diff --git a/conf/genotoul.config b/conf/genotoul.config new file mode 100644 index 0000000..588028d --- /dev/null +++ b/conf/genotoul.config @@ -0,0 +1,27 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Genotoul cluster profile' + config_profile_contact = 'support.bioinfo.genotoul@inra.fr' + config_profile_url = 'http://bioinfo.genotoul.fr/' +} + +singularity { + // need one image per execution + enabled = true + runOptions = '-B /bank -B /work2 -B /work -B /save -B /home' + +} + +process { + executor = 'slurm' +} + +params { + saveReference = true + igenomesIgnore = true + // Max resources requested by a normal node on genotoul. + max_memory = 120.GB + max_cpus = 48 + max_time = 96.h +} + diff --git a/docs/genotoul.md b/docs/genotoul.md new file mode 100644 index 0000000..026a59c --- /dev/null +++ b/docs/genotoul.md @@ -0,0 +1,85 @@ + + + +# nf-core/configs: Bioinfo Genotoul Configuration + +All nf-core pipelines have been successfully configured for use on the Bioinfo Genotoul cluster at the INRA toulouse. + +To use, run the pipeline with `-profile genotoul`. This will download and +launch the [`genotoul.config`](../conf/genotoul.config) which has been +pre-configured with a setup suitable for the Bioinfo Genotoul cluster. + +Using this profile, a docker image containing all of the required software +will be downloaded, and converted to a Singularity image before execution +of the pipeline. Images are stored for all users in following directory `/usr/local/bioinfo/src/NextflowWorkflows/singularity-img/`. + +## Running the workflow ib the Genologin cluster. + +Before running the pipeline you will need to load Nextflow and +Singularity using the environment module system on Genotoul. You can do +this by issuing the commands below: + + +Once connected on our frontal node : +``` +# Login to a compute node +srun --mem=4G --pty bash +``` + +Setup default nextflow and singularity home directory (to be done only one time): +``` +sh /usr/local/bioinfo/src/NextflowWorkflows/create_nfx_dirs.sh +``` + +Load environment : +``` +module purge +module load bioinfo/nfcore-Nextflow-v19.04.0 +``` + +Try a test workflow (for example the methylseq workflow) : +``` +nextflow run nf-core/methylseq -profile genotoul,test +``` + +Create launch script `nfcore-rnaseq.sh` : +``` +#!/bin/bash +#SBATCH -p workq +#SBATCH -t 1:00:00 #time in hour +#SBATCH --mem=4G +#SBATCH --mail-type=BEGIN,END,FAIL + +module load bioinfo/nfcore-Nextflow-v19.04.0 +nextflow run nf-core/methylseq -profile genotoul,test +``` +Launch on the cluster with sbatch: +``` +sbatch nfcore-rnaseq.sh +``` + +## Mounted directory + +By default, available mount points are : + - /bank + - /home + - /save + - /work + - /work2 + + +To have access to specific other mount point (such as nosave or project) +you can add a config profile file with option `-profile` and wich contain: +``` +singularity.runOptions = '-B /diretory/to/mount' +``` + +## Databanks + +A local copy of several genomes are available in `/bank` directory. See +our [databank page](http://bioinfo.genotoul.fr/index.php/resources-2/databanks/) +to search for your favorite genome. + + +>NB: You will need an account to use the HPC cluster on Genotoul in order +to run the pipeline. If in doubt see http://bioinfo.genotoul.fr/. From 42d968fd2a9274a457dcbc80dea61a0bc5170228 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 16:23:25 +0100 Subject: [PATCH 02/27] add genotoul in nfcore_custom --- nfcore_custom.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nfcore_custom.config b/nfcore_custom.config index 06182f5..017d326 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -21,6 +21,7 @@ profiles { crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } hebbe { includeConfig "${params.custom_config_base}/conf/hebbe.config" } @@ -44,6 +45,7 @@ params { // This is a groovy map, not a nextflow parameter set hostnames = [ crick: ['.thecrick.org'], + genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] ] From 9e5b76827fc7e3c6dafc1af8a2b69be068bf0761 Mon Sep 17 00:00:00 2001 From: noirot Date: Mon, 25 Nov 2019 16:37:00 +0100 Subject: [PATCH 03/27] add genotoul in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b9812b6..cb7e3d4 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ Currently documentation is available for the following systems: * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) +* [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) From 42d99e700666972ad03dc8fda0b81784cc9d8eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 13:18:34 +0100 Subject: [PATCH 04/27] Update genotoul.md --- docs/genotoul.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 026a59c..47e1916 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -1,6 +1,3 @@ - - - # nf-core/configs: Bioinfo Genotoul Configuration All nf-core pipelines have been successfully configured for use on the Bioinfo Genotoul cluster at the INRA toulouse. From dea11df453a331397c4da073f14123240b55c4b4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:07:25 +0100 Subject: [PATCH 05/27] Both on PR and on push --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..76b0bf4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: Configs tests +on: [pull_request, push] jobs: profile_test: runs-on: ubuntu-latest From c3c7c74d79de2ef8a401123af4b917d456d2b854 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:11:16 +0100 Subject: [PATCH 06/27] Better code --- .github/workflows/main.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 76b0bf4..e60ec19 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,18 @@ name: Configs tests on: [pull_request, push] jobs: + test_all_profiles: + runs-on: ubuntu-latest + name: Check if all profiles are tested + steps: + - uses: actions/checkout@v1 + - name: Check whether profiles are all tested + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest name: Run individual profiles + needs: test_all_profiles strategy: matrix: profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] @@ -14,8 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested - run: | - python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 0afe00ded5de63e3928c4f54c329046326701a73 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:19 +0100 Subject: [PATCH 07/27] Generalize name --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e60ec19..85ebe2a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest - name: Run individual profiles + name: Run ${{ matrix.profile }} profile needs: test_all_profiles strategy: matrix: From 47ac2085f7945e6c019b8e3579d2ab75f316198e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:12:41 +0100 Subject: [PATCH 08/27] Better names --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 85ebe2a..434504f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,5 @@ jobs: run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: "Check profile" + - name: Check ${{ matrix.profile }} profile run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 2ebd2180edb3543ad404efb16c2793e4d4a2fbf4 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:27:22 +0100 Subject: [PATCH 09/27] Add in SCRATCH env --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 434504f..c3101f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,4 +24,6 @@ jobs: wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - name: Check ${{ matrix.profile }} profile + env: + SCRATCH: '~' run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 37c6714cc75ab25ac4d4835228ca286f4eb93538 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:29:52 +0100 Subject: [PATCH 10/27] Nice prince --- conf/prince.config | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/conf/prince.config b/conf/prince.config index 1c2ea2c..c665d9f 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -18,10 +18,6 @@ singularity { } process { - beforeScript = """ - module load $singularityModule - module load $squashfsModule - """ - .stripIndent() + beforeScript = "module load $singularityModule $squashfsModule" executor = 'slurm' } From bac91d03faacef910b0d90b1daa1abcc41a2c403 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 14:38:57 +0100 Subject: [PATCH 11/27] Ignore czi stuff for now --- .github/workflows/main.yml | 2 +- bin/cchecker.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3101f3..3288efd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow diff --git a/bin/cchecker.py b/bin/cchecker.py index 029e6d5..eb995fc 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -48,6 +48,9 @@ def check_config(Config, Github): ###Check Github Config now tests = set() + ###Ignore these profiles + ignore_me = ['czbiohub_aws_highpriority', 'czbiohub_aws'] + tests.update(ignore_me) with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line): From 42c949db95947490733d5aec4c150760102006ef Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:43:42 +0100 Subject: [PATCH 12/27] correct md to pass linter --- docs/genotoul.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 47e1916..831e90e 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -10,37 +10,41 @@ Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. Images are stored for all users in following directory `/usr/local/bioinfo/src/NextflowWorkflows/singularity-img/`. -## Running the workflow ib the Genologin cluster. +## Running the workflow ib the Genologin cluster Before running the pipeline you will need to load Nextflow and Singularity using the environment module system on Genotoul. You can do this by issuing the commands below: - Once connected on our frontal node : -``` + +```bash # Login to a compute node srun --mem=4G --pty bash ``` Setup default nextflow and singularity home directory (to be done only one time): -``` + +```bash sh /usr/local/bioinfo/src/NextflowWorkflows/create_nfx_dirs.sh ``` Load environment : -``` + +```bash module purge module load bioinfo/nfcore-Nextflow-v19.04.0 ``` Try a test workflow (for example the methylseq workflow) : -``` + +```bash nextflow run nf-core/methylseq -profile genotoul,test ``` Create launch script `nfcore-rnaseq.sh` : -``` + +```bash #!/bin/bash #SBATCH -p workq #SBATCH -t 1:00:00 #time in hour @@ -51,7 +55,8 @@ module load bioinfo/nfcore-Nextflow-v19.04.0 nextflow run nf-core/methylseq -profile genotoul,test ``` Launch on the cluster with sbatch: -``` + +```bash sbatch nfcore-rnaseq.sh ``` @@ -64,10 +69,9 @@ By default, available mount points are : - /work - /work2 - To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and wich contain: -``` +```bash singularity.runOptions = '-B /diretory/to/mount' ``` @@ -77,6 +81,5 @@ A local copy of several genomes are available in `/bank` directory. See our [databank page](http://bioinfo.genotoul.fr/index.php/resources-2/databanks/) to search for your favorite genome. - >NB: You will need an account to use the HPC cluster on Genotoul in order -to run the pipeline. If in doubt see http://bioinfo.genotoul.fr/. +to run the pipeline. If in doubt see [http://bioinfo.genotoul.fr/](http://bioinfo.genotoul.fr/). From e95196451fe850201e0b471e2b3cc81d78e210b3 Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:47:23 +0100 Subject: [PATCH 13/27] correct md to pass linter --- docs/genotoul.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 831e90e..5f46f21 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -54,6 +54,7 @@ Create launch script `nfcore-rnaseq.sh` : module load bioinfo/nfcore-Nextflow-v19.04.0 nextflow run nf-core/methylseq -profile genotoul,test ``` + Launch on the cluster with sbatch: ```bash @@ -63,14 +64,15 @@ sbatch nfcore-rnaseq.sh ## Mounted directory By default, available mount points are : - - /bank - - /home - - /save - - /work - - /work2 +- /bank +- /home +- /save +- /work +- /work2 To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and wich contain: + ```bash singularity.runOptions = '-B /diretory/to/mount' ``` From dd4ff452dac7efc00fd04501a460656b1595272f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:49:47 +0100 Subject: [PATCH 14/27] Update conf/genotoul.config Co-Authored-By: Maxime Garcia --- conf/genotoul.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conf/genotoul.config b/conf/genotoul.config index 588028d..e5a1242 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -18,10 +18,9 @@ process { params { saveReference = true - igenomesIgnore = true + igenomes_ignore = true // Max resources requested by a normal node on genotoul. max_memory = 120.GB max_cpus = 48 max_time = 96.h } - From 611c318569c4d9d4683052ec1d8009c41d05f981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:50:05 +0100 Subject: [PATCH 15/27] Update conf/genotoul.config Co-Authored-By: Maxime Garcia --- conf/genotoul.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/genotoul.config b/conf/genotoul.config index e5a1242..75ad2c4 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -17,7 +17,7 @@ process { } params { - saveReference = true + save_reference = true igenomes_ignore = true // Max resources requested by a normal node on genotoul. max_memory = 120.GB From 288294994e0c9df795cc34c71464ae8a40663e9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:52:32 +0100 Subject: [PATCH 16/27] Update docs/genotoul.md ok Co-Authored-By: Maxime Garcia --- docs/genotoul.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 5f46f21..e6b2f13 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -71,7 +71,7 @@ By default, available mount points are : - /work2 To have access to specific other mount point (such as nosave or project) -you can add a config profile file with option `-profile` and wich contain: +you can add a config profile file with option `-profile` and which contain: ```bash singularity.runOptions = '-B /diretory/to/mount' From 3ed206b7a773efdde19e6291e41cd7ecb6e40f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 14:52:45 +0100 Subject: [PATCH 17/27] Update docs/genotoul.md ok Co-Authored-By: Maxime Garcia --- docs/genotoul.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index e6b2f13..5fe02b7 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -74,7 +74,7 @@ To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and which contain: ```bash -singularity.runOptions = '-B /diretory/to/mount' +singularity.runOptions = '-B /directory/to/mount' ``` ## Databanks From 3d8da62db3c29d0c631f6167112208e3fdd0972a Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 14:57:17 +0100 Subject: [PATCH 18/27] add genotoul profile name to the build matrix --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..6d3f127 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,7 +6,7 @@ jobs: name: Run individual profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow @@ -17,4 +17,4 @@ jobs: run: | python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} From 07b1f17d6a81a145d3c1afde1cf8ebd53b746a4a Mon Sep 17 00:00:00 2001 From: noirot Date: Tue, 26 Nov 2019 15:02:06 +0100 Subject: [PATCH 19/27] correct md to pass linter --- docs/genotoul.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/genotoul.md b/docs/genotoul.md index 5fe02b7..b86cf66 100644 --- a/docs/genotoul.md +++ b/docs/genotoul.md @@ -63,12 +63,13 @@ sbatch nfcore-rnaseq.sh ## Mounted directory -By default, available mount points are : -- /bank -- /home -- /save -- /work -- /work2 +By default, available mount points are: + +* /bank +* /home +* /save +* /work +* /work2 To have access to specific other mount point (such as nosave or project) you can add a config profile file with option `-profile` and which contain: From 0fdcf8c70c86b18c648cbf303b02514a6fa0cda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noirot=20C=C3=A9line?= Date: Tue, 26 Nov 2019 15:10:13 +0100 Subject: [PATCH 20/27] Update .github/workflows/main.yml Co-Authored-By: Alexander Peltzer --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e36b79a..a9d41b5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] steps: - uses: actions/checkout@v1 - name: Install Nextflow From 9cb5137ef225888e5e466d2a339c5b78f07db5e0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 15:12:42 +0100 Subject: [PATCH 21/27] Better description --- README.md | 2 +- bin/cchecker.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f060b03..58a65d9 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Currently documentation is available for the following systems: [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. -Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. +Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to the alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. If you forget to do this, tests will fail and complain about that. Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. diff --git a/bin/cchecker.py b/bin/cchecker.py index eb995fc..4290170 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -64,7 +64,7 @@ def check_config(Config, Github): sys.exit(0) else: #Maybe report what is missing here too - print("Tests don't seem to test these profiles properly!\n") + print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n") print(config_profiles.symmetric_difference(tests)) sys.exit(1) From ca6ddcadb07d44725c12c81af27f86f326753eed Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:25:35 +0100 Subject: [PATCH 22/27] Matching max CPUs to SDAG (supercruncher8) --- conf/shh.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/shh.config b/conf/shh.config index 14da627..8a7f812 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -23,8 +23,8 @@ executor { params { max_memory = 2.TB - max_cpus = 32 + max_cpus = 128 max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" -} \ No newline at end of file +} From c3d27099ca02436fc3462504dfddfbfd4ceb3961 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:26:05 +0100 Subject: [PATCH 23/27] Updated authors --- conf/shh.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/shh.config b/conf/shh.config index 8a7f812..424a0ee 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,7 +1,7 @@ //Profile config names for nf-core/configs params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' - config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' } From 12ec22838c5a23f74940b2912aa98bd3e3a78419 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:29:03 +0100 Subject: [PATCH 24/27] Specified max sources currently at SDAG limits (exceeds cdag) --- docs/shh.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/shh.md b/docs/shh.md index 78abed0..ec268b5 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -11,7 +11,11 @@ To use, run the pipeline with `-profile shh`. This will download and launch the however this will likely change to a read-only directory in the future that will be managed by IT. This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. + Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. + >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. + +>NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. From 89506a653cbe6edbbab5459febfae40ffe9f9cff Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 26 Nov 2019 15:34:25 +0100 Subject: [PATCH 25/27] Remove empty lines in block quotes for linting --- docs/shh.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/shh.md b/docs/shh.md index ec268b5..c89d550 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -15,7 +15,5 @@ This configuration will automatically choose the correct SLURM queue (`short`,`m Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. - >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. - >NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. From da922b80059f7432737aa50f8140910bd9507f23 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 15:45:33 +0100 Subject: [PATCH 26/27] Fix README --- README.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ec810d7..2b18dcc 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ -# [![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) +# [![nf-core/configs](docs/images/nfcore-configs_logo.png "nf-core/configs")](https://github.com/nf-core/configs) -[![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) +[![Lint Status](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [![nf-core/configs](https://github.com/nf-core/configs)](#nf-coreconfigshttpsgithubcomnf-coreconfigs) - * [Table of contents](#table-of-contents) - * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) - * [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) - * [Help](#help) +* [Table of contents](#table-of-contents) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) +* [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Help](#help) ## Using an existing config From b756ce54a488e2f65a63830fb88c6289f98b3426 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 26 Nov 2019 16:09:08 +0100 Subject: [PATCH 27/27] Add genotoul fix --- conf/genotoul.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/genotoul.config b/conf/genotoul.config index 75ad2c4..8ad50b1 100644 --- a/conf/genotoul.config +++ b/conf/genotoul.config @@ -19,6 +19,7 @@ process { params { save_reference = true igenomes_ignore = true + igenomesIgnore = true //deprecated // Max resources requested by a normal node on genotoul. max_memory = 120.GB max_cpus = 48