diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9c2f12..4ff3730 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,20 +1,30 @@ name: Configs tests +on: [pull_request, push] jobs: + test_all_profiles: + runs-on: ubuntu-latest + name: Check if all profiles are tested + steps: + - uses: actions/checkout@v1 + - name: Check whether profiles are all tested + run: | + python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml profile_test: runs-on: ubuntu-latest - name: Run individual profiles + name: Run ${{ matrix.profile }} profile + needs: test_all_profiles strategy: matrix: - profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'czbiohub_aws_highpriority', 'czbiohub_aws', 'denbi_qbic', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + profile: ['awsbatch', 'bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga', 'cfc', 'crick', 'denbi_qbic', 'genotoul', 'genouest', 'gis', 'hebbe', 'kraken', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hex', 'uppmax_devel', 'uppmax', 'uzh'] + steps: - uses: actions/checkout@v1 - name: Install Nextflow run: | wget -qO- get.nextflow.io | bash sudo mv nextflow /usr/local/bin/ - - name: Check whether profiles are all tested - run: | - python ${GITHUB_WORKSPACE}/bin/cchecker.py ${GITHUB_WORKSPACE}/nfcore_custom.config ${GITHUB_WORKSPACE}/.github/workflows/main.yml - - name: "Check profile" - run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} \ No newline at end of file + - name: Check ${{ matrix.profile }} profile + env: + SCRATCH: '~' + run: nextflow run ${GITHUB_WORKSPACE}/configtest.nf --custom_config_base=${GITHUB_WORKSPACE} -profile ${{ matrix.profile }} diff --git a/README.md b/README.md index f060b03..2b18dcc 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,21 @@ -# [![nf-core/configs](docs/images/nfcore-configs_logo.png)](https://github.com/nf-core/configs) +# [![nf-core/configs](docs/images/nfcore-configs_logo.png "nf-core/configs")](https://github.com/nf-core/configs) -[![Lint Status](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/nfcore%20configs%20tests/badge.svg) +[![Lint Status](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg)](https://github.com/nf-core/configs/workflows/Configs%20tests/badge.svg) A repository for hosting nextflow config files containing custom parameters required to run nf-core pipelines at different Institutions. ## Table of contents -* [![nf-core/configs](https://github.com/nf-core/configs)](#nf-coreconfigshttpsgithubcomnf-coreconfigs) - * [Table of contents](#table-of-contents) - * [Using an existing config](#using-an-existing-config) - * [Configuration and parameters](#configuration-and-parameters) - * [Offline usage](#offline-usage) - * [Adding a new config](#adding-a-new-config) - * [Checking user hostnames](#checking-user-hostnames) - * [Testing](#testing) - * [Documentation](#documentation) - * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) - * [Help](#help) +* [Table of contents](#table-of-contents) +* [Using an existing config](#using-an-existing-config) + * [Configuration and parameters](#configuration-and-parameters) + * [Offline usage](#offline-usage) +* [Adding a new config](#adding-a-new-config) +* [Checking user hostnames](#checking-user-hostnames) + * [Testing](#testing) + * [Documentation](#documentation) + * [Uploading to `nf-core/configs`](#uploading-to-nf-coreconfigs) +* [Help](#help) ## Using an existing config @@ -97,6 +96,7 @@ Currently documentation is available for the following systems: * [CZBIOHUB_AWS](docs/czbiohub.md) * [CZBIOHUB_AWS_HIGHPRIORITY](docs/czbiohub.md) * [DENBI_QBIC](docs/denbi_qbic.md) +* [GENOTOUL](docs/genotoul.md) * [GENOUEST](docs/genouest.md) * [GIS](docs/gis.md) * [HEBBE](docs/hebbe.md) @@ -115,7 +115,7 @@ Currently documentation is available for the following systems: [Fork](https://help.github.com/articles/fork-a-repo/) the `nf-core/configs` repository to your own GitHub account. Within the local clone of your fork add the custom config file to the [`conf/`](https://github.com/nf-core/configs/tree/master/conf) directory, and the documentation file to the [`docs/`](https://github.com/nf-core/configs/tree/master/docs) directory. You will also need to edit and add your custom profile to the [`nfcore_custom.config`](https://github.com/nf-core/configs/blob/master/nfcore_custom.config) file in the top-level directory of the clone. -Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. +Afterwards, make sure to edit the `.github/main.yml` file and add your profile name to the alphabetically sorted `profile:` scope. This way, it will be tested automatically using GitHub Actions. If you forget to do this, tests will fail and complain about that. Commit and push these changes to your local clone on GitHub, and then [create a pull request](https://help.github.com/articles/creating-a-pull-request-from-a-fork/) on the `nf-core/configs` GitHub repo with the appropriate information. diff --git a/bin/cchecker.py b/bin/cchecker.py index 029e6d5..4290170 100644 --- a/bin/cchecker.py +++ b/bin/cchecker.py @@ -48,6 +48,9 @@ def check_config(Config, Github): ###Check Github Config now tests = set() + ###Ignore these profiles + ignore_me = ['czbiohub_aws_highpriority', 'czbiohub_aws'] + tests.update(ignore_me) with open(Github, 'r') as ghfile: for line in ghfile: if re.search('profile: ', line): @@ -61,7 +64,7 @@ def check_config(Config, Github): sys.exit(0) else: #Maybe report what is missing here too - print("Tests don't seem to test these profiles properly!\n") + print("Tests don't seem to test these profiles properly. Please check whether you added the profile to the Github Actions testing YAML.\n") print(config_profiles.symmetric_difference(tests)) sys.exit(1) diff --git a/conf/genotoul.config b/conf/genotoul.config new file mode 100644 index 0000000..8ad50b1 --- /dev/null +++ b/conf/genotoul.config @@ -0,0 +1,27 @@ +//Profile config names for nf-core/configs +params { + config_profile_description = 'The Genotoul cluster profile' + config_profile_contact = 'support.bioinfo.genotoul@inra.fr' + config_profile_url = 'http://bioinfo.genotoul.fr/' +} + +singularity { + // need one image per execution + enabled = true + runOptions = '-B /bank -B /work2 -B /work -B /save -B /home' + +} + +process { + executor = 'slurm' +} + +params { + save_reference = true + igenomes_ignore = true + igenomesIgnore = true //deprecated + // Max resources requested by a normal node on genotoul. + max_memory = 120.GB + max_cpus = 48 + max_time = 96.h +} diff --git a/conf/prince.config b/conf/prince.config index 1c2ea2c..c665d9f 100644 --- a/conf/prince.config +++ b/conf/prince.config @@ -18,10 +18,6 @@ singularity { } process { - beforeScript = """ - module load $singularityModule - module load $squashfsModule - """ - .stripIndent() + beforeScript = "module load $singularityModule $squashfsModule" executor = 'slurm' } diff --git a/conf/shh.config b/conf/shh.config index 14da627..424a0ee 100644 --- a/conf/shh.config +++ b/conf/shh.config @@ -1,7 +1,7 @@ //Profile config names for nf-core/configs params { config_profile_description = 'MPI SHH cluster profile provided by nf-core/configs.' - config_profile_contact = 'James Fellows Yates (@jfy133)' + config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)' config_profile_url = 'https://shh.mpg.de' } @@ -23,8 +23,8 @@ executor { params { max_memory = 2.TB - max_cpus = 32 + max_cpus = 128 max_time = 720.h //Illumina iGenomes reference file path igenomes_base = "/projects1/public_data/igenomes/" -} \ No newline at end of file +} diff --git a/docs/genotoul.md b/docs/genotoul.md new file mode 100644 index 0000000..b86cf66 --- /dev/null +++ b/docs/genotoul.md @@ -0,0 +1,88 @@ +# nf-core/configs: Bioinfo Genotoul Configuration + +All nf-core pipelines have been successfully configured for use on the Bioinfo Genotoul cluster at the INRA toulouse. + +To use, run the pipeline with `-profile genotoul`. This will download and +launch the [`genotoul.config`](../conf/genotoul.config) which has been +pre-configured with a setup suitable for the Bioinfo Genotoul cluster. + +Using this profile, a docker image containing all of the required software +will be downloaded, and converted to a Singularity image before execution +of the pipeline. Images are stored for all users in following directory `/usr/local/bioinfo/src/NextflowWorkflows/singularity-img/`. + +## Running the workflow ib the Genologin cluster + +Before running the pipeline you will need to load Nextflow and +Singularity using the environment module system on Genotoul. You can do +this by issuing the commands below: + +Once connected on our frontal node : + +```bash +# Login to a compute node +srun --mem=4G --pty bash +``` + +Setup default nextflow and singularity home directory (to be done only one time): + +```bash +sh /usr/local/bioinfo/src/NextflowWorkflows/create_nfx_dirs.sh +``` + +Load environment : + +```bash +module purge +module load bioinfo/nfcore-Nextflow-v19.04.0 +``` + +Try a test workflow (for example the methylseq workflow) : + +```bash +nextflow run nf-core/methylseq -profile genotoul,test +``` + +Create launch script `nfcore-rnaseq.sh` : + +```bash +#!/bin/bash +#SBATCH -p workq +#SBATCH -t 1:00:00 #time in hour +#SBATCH --mem=4G +#SBATCH --mail-type=BEGIN,END,FAIL + +module load bioinfo/nfcore-Nextflow-v19.04.0 +nextflow run nf-core/methylseq -profile genotoul,test +``` + +Launch on the cluster with sbatch: + +```bash +sbatch nfcore-rnaseq.sh +``` + +## Mounted directory + +By default, available mount points are: + +* /bank +* /home +* /save +* /work +* /work2 + +To have access to specific other mount point (such as nosave or project) +you can add a config profile file with option `-profile` and which contain: + +```bash +singularity.runOptions = '-B /directory/to/mount' +``` + +## Databanks + +A local copy of several genomes are available in `/bank` directory. See +our [databank page](http://bioinfo.genotoul.fr/index.php/resources-2/databanks/) +to search for your favorite genome. + +>NB: You will need an account to use the HPC cluster on Genotoul in order +to run the pipeline. If in doubt see [http://bioinfo.genotoul.fr/](http://bioinfo.genotoul.fr/). diff --git a/docs/shh.md b/docs/shh.md index 78abed0..c89d550 100644 --- a/docs/shh.md +++ b/docs/shh.md @@ -11,7 +11,9 @@ To use, run the pipeline with `-profile shh`. This will download and launch the however this will likely change to a read-only directory in the future that will be managed by IT. This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`,`supercruncher`) depending on the time and memory required by each process. + Please note that there is no `supercruncher` queue on CDAG. >NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact IT. >NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT. +>NB: The maximum CPUs/Mem are currently adapted for SDAG resource maximums - i.e. will exceed CDAG. Be careful when running larges jobs that error-retries may exceed limits and get 'stuck' in SLURM. diff --git a/nfcore_custom.config b/nfcore_custom.config index 740fcc5..90145b3 100644 --- a/nfcore_custom.config +++ b/nfcore_custom.config @@ -22,8 +22,10 @@ profiles { cfc { includeConfig "${params.custom_config_base}/conf/cfc.config" } crick { includeConfig "${params.custom_config_base}/conf/crick.config" } czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" } - czbiohub_aws_highpriority { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; - includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config" } + czbiohub_aws_highpriority { + includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config"; + includeConfig "${params.custom_config_base}/conf/czbiohub_aws_highpriority.config"} + genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" } denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" } genouest { includeConfig "${params.custom_config_base}/conf/genouest.config" } gis { includeConfig "${params.custom_config_base}/conf/gis.config" } @@ -48,6 +50,7 @@ params { // This is a groovy map, not a nextflow parameter set hostnames = [ crick: ['.thecrick.org'], + genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'], genouest: ['.genouest.org'], uppmax: ['.uppmax.uu.se'] ]