mirror of
https://github.com/MillironX/nf-configs.git
synced 2024-11-22 08:29:54 +00:00
Merge branch 'master' into master
This commit is contained in:
commit
bfab371c94
11 changed files with 178 additions and 3 deletions
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
|
@ -16,7 +16,7 @@ jobs:
|
||||||
needs: test_all_profiles
|
needs: test_all_profiles
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh']
|
profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'ifb_core', 'imperial', 'imperial_mb', 'jax', 'kraken', 'mpcdf', 'munin', 'oist', 'pasteur', 'phoenix', 'prince', 'seg_globe', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh']
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v1
|
- uses: actions/checkout@v1
|
||||||
- name: Install Nextflow
|
- name: Install Nextflow
|
||||||
|
|
|
@ -113,6 +113,7 @@ Currently documentation is available for the following systems:
|
||||||
* [GOOGLE](docs/google.md)
|
* [GOOGLE](docs/google.md)
|
||||||
* [HEBBE](docs/hebbe.md)
|
* [HEBBE](docs/hebbe.md)
|
||||||
* [ICR_DAVROS](docs/icr_davros.md)
|
* [ICR_DAVROS](docs/icr_davros.md)
|
||||||
|
* [JAX](docs/jax.md)
|
||||||
* [KRAKEN](docs/kraken.md)
|
* [KRAKEN](docs/kraken.md)
|
||||||
* [MPCDF](docs/mpcdf.md)
|
* [MPCDF](docs/mpcdf.md)
|
||||||
* [MUNIN](docs/munin.md)
|
* [MUNIN](docs/munin.md)
|
||||||
|
|
|
@ -2,13 +2,13 @@
|
||||||
params {
|
params {
|
||||||
config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs'
|
config_profile_description = 'CLIP BATCH ENVIRONMENT (CBE) cluster profile provided by nf-core/configs'
|
||||||
config_profile_contact = 'Patrick Hüther (@phue)'
|
config_profile_contact = 'Patrick Hüther (@phue)'
|
||||||
config_profile_url = 'http://www.gmi.oeaw.ac.at/'
|
config_profile_url = 'https://clip.science'
|
||||||
}
|
}
|
||||||
|
|
||||||
process {
|
process {
|
||||||
executor = 'slurm'
|
executor = 'slurm'
|
||||||
queue = { task.memory <= 170.GB ? 'c' : 'm' }
|
queue = { task.memory <= 170.GB ? 'c' : 'm' }
|
||||||
clusterOptions = { task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' }
|
clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' }
|
||||||
module = 'anaconda3/2019.10'
|
module = 'anaconda3/2019.10'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
24
conf/ifb_core.config
Normal file
24
conf/ifb_core.config
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
//Profile config names for nf-core/configs
|
||||||
|
params {
|
||||||
|
config_profile_description = 'The IFB core cluster profile'
|
||||||
|
config_profile_contact = 'https://community.france-bioinformatique.fr'
|
||||||
|
config_profile_url = 'https://www.france-bioinformatique.fr/'
|
||||||
|
}
|
||||||
|
|
||||||
|
singularity {
|
||||||
|
// need one image per execution
|
||||||
|
enabled = true
|
||||||
|
runOptions = '-B /shared'
|
||||||
|
}
|
||||||
|
|
||||||
|
process {
|
||||||
|
executor = 'slurm'
|
||||||
|
}
|
||||||
|
|
||||||
|
params {
|
||||||
|
igenomes_ignore = true
|
||||||
|
// Max resources requested by a normal node on genotoul.
|
||||||
|
max_memory = 240.GB
|
||||||
|
max_cpus = 28
|
||||||
|
max_time = 96.h
|
||||||
|
}
|
21
conf/jax.config
Normal file
21
conf/jax.config
Normal file
|
@ -0,0 +1,21 @@
|
||||||
|
params {
|
||||||
|
config_profile_description = 'The Jackson Laboratory Sumner HPC profile provided by nf-core/configs.'
|
||||||
|
config_profile_contact = 'Asaf Peer (@peera)'
|
||||||
|
config_profile_url = 'https://jacksonlaboratory.sharepoint.com/sites/ResearchIT/SitePages/Welcome-to-Sumner.aspx'
|
||||||
|
}
|
||||||
|
|
||||||
|
executor.$slurm.queueSize = 250
|
||||||
|
process {
|
||||||
|
executor = "slurm"
|
||||||
|
queue = "compute"
|
||||||
|
clusterOptions = {task.time < 72.h ? '-q batch' : '-q long'}
|
||||||
|
module = "slurm"
|
||||||
|
beforeScript = 'module load singularity'
|
||||||
|
}
|
||||||
|
singularity.enabled = true
|
||||||
|
singularity.autoMounts = true
|
||||||
|
params {
|
||||||
|
max_memory = 768.GB
|
||||||
|
max_cpus = 70
|
||||||
|
max_time = 336.h
|
||||||
|
}
|
|
@ -5,6 +5,11 @@ params {
|
||||||
config_profile_url = 'http://docs.oithpc.utdallas.edu/'
|
config_profile_url = 'http://docs.oithpc.utdallas.edu/'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
env {
|
||||||
|
TMPDIR = '/home/$USER/scratch/tmp'
|
||||||
|
SINGULARITY_CACHEDIR = '/home/$USER/scratch/tmp'
|
||||||
|
}
|
||||||
|
|
||||||
singularity {
|
singularity {
|
||||||
enabled = true
|
enabled = true
|
||||||
envWhitelist='SINGULARITY_BINDPATH'
|
envWhitelist='SINGULARITY_BINDPATH'
|
||||||
|
@ -15,6 +20,28 @@ process {
|
||||||
beforeScript = 'module load singularity/3.2.1'
|
beforeScript = 'module load singularity/3.2.1'
|
||||||
executor = 'slurm'
|
executor = 'slurm'
|
||||||
queue = { task.memory >= 32.GB && task.cpu <= 12 ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' }
|
queue = { task.memory >= 32.GB && task.cpu <= 12 ? 'Kim': task.memory <= 24.GB && task.cpu <= 8 ? 'smallmem' : 'genomics' }
|
||||||
|
|
||||||
|
withName:TRIMGALORE {
|
||||||
|
memory = 31.GB
|
||||||
|
}
|
||||||
|
|
||||||
|
withLabel:process_low {
|
||||||
|
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
|
||||||
|
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
|
||||||
|
time = { check_max( 6.h * task.attempt, 'time' ) }
|
||||||
|
}
|
||||||
|
|
||||||
|
withLabel:process_medium {
|
||||||
|
cpus = { check_max( 16 * task.attempt, 'cpus' ) }
|
||||||
|
memory = { check_max( 31.GB * task.attempt, 'memory' ) }
|
||||||
|
time = { check_max( 8.h * task.attempt, 'time' ) }
|
||||||
|
}
|
||||||
|
|
||||||
|
withLabel:process_high {
|
||||||
|
cpus = { check_max( 12 * task.attempt, 'cpus' ) }
|
||||||
|
memory = { check_max( 120.GB * task.attempt, 'memory' ) }
|
||||||
|
time = { check_max( 16.h * task.attempt, 'time' ) }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
params {
|
params {
|
||||||
|
|
28
conf/wcm.config
Normal file
28
conf/wcm.config
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
singularityDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow"
|
||||||
|
|
||||||
|
params {
|
||||||
|
config_profile_description = 'Weill Cornell Medicine, Scientific Computing Unit Slurm cluster profile provided by nf-core/configs'
|
||||||
|
config_profile_contact = 'Ashley Stephen Doane, PhD (@DoaneAS)'
|
||||||
|
igenomes_base = '/athena/elementolab/scratch/reference/igenomes'
|
||||||
|
}
|
||||||
|
|
||||||
|
singularity {
|
||||||
|
enabled = true
|
||||||
|
envWhitelist='SINGULARITY_BINDPATH'
|
||||||
|
cacheDir = "/athena/elementolab/scratch/reference/.singularity/singularity_images_nextflow"
|
||||||
|
autoMounts = true
|
||||||
|
}
|
||||||
|
|
||||||
|
process {
|
||||||
|
executor = 'slurm'
|
||||||
|
queue = 'panda_physbio'
|
||||||
|
scratch = true
|
||||||
|
scratch = '/scratchLocal/`whoami`_${SLURM_JOBID}'
|
||||||
|
}
|
||||||
|
|
||||||
|
params {
|
||||||
|
max_memory = 32.GB
|
||||||
|
max_cpus = 8
|
||||||
|
max_time = 24.h
|
||||||
|
}
|
||||||
|
|
40
docs/ifb_core.md
Normal file
40
docs/ifb_core.md
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
# nf-core/configs: IFB core Configuration
|
||||||
|
|
||||||
|
All nf-core pipelines have been successfully configured for use on the cluster of the IFB (Institut Francais de Bioinformatique).
|
||||||
|
|
||||||
|
To use, run the pipeline with `-profile ifb_core`. This will download and launch the [`ifb_core.config`](../conf/ifb_core.config) which has been pre-configured with a setup suitable for the IFB core cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline.
|
||||||
|
|
||||||
|
## How to use on IFB core
|
||||||
|
|
||||||
|
Before running the pipeline you will need to load Nextflow using the environment module system on IFB core. You can do this by issuing the commands below:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Login to a compute node
|
||||||
|
srun --pty bash
|
||||||
|
|
||||||
|
## Load Nextflow and Singularity environment modules
|
||||||
|
module purge
|
||||||
|
module load nextflow/20.04.1
|
||||||
|
|
||||||
|
# Run a downloaded/git-cloned nextflow workflow from
|
||||||
|
nextflow run \\
|
||||||
|
nf-core/workflow \\
|
||||||
|
-resume
|
||||||
|
-profile ifb_core \\
|
||||||
|
--email my-email@example.org \\
|
||||||
|
-c my-specific.config
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
# Or use the nf-core client
|
||||||
|
nextflow run nf-core/rnaseq ...
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
## Databanks
|
||||||
|
|
||||||
|
A local copy of several genomes are available in `/shared/bank` directory. See
|
||||||
|
our [databank page](https://ifb-elixirfr.gitlab.io/cluster/doc/banks/)
|
||||||
|
to search for your favorite genome.
|
||||||
|
|
||||||
|
>NB: You will need an account to use the HPC cluster on IFB core in order to run the pipeline. If in doubt contact IT or go to [account page](https://my.cluster.france-bioinformatique.fr/manager2/login).
|
8
docs/jax.md
Normal file
8
docs/jax.md
Normal file
|
@ -0,0 +1,8 @@
|
||||||
|
# nf-core/configs: JAX Configuration
|
||||||
|
|
||||||
|
All nf-core pipelines have been successfully configured for use on the JAX Sumner cluster at The Jackson Laboratory.
|
||||||
|
|
||||||
|
To use, run the pipeline with `-profile jax`. This will download and launch the [`jax.config`](../conf/jax.config) which has been pre-configured with a setup suitable for JAX Sumner cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline and slurm will be used as well.
|
||||||
|
|
||||||
|
>NB: You will need an account to use the HPC cluster JAX in order to run the pipeline. If in doubt contact IT.
|
||||||
|
>NB: Nextflow should not be executed on the login nodes. If in doubt contact IT.
|
24
docs/wcm.md
Normal file
24
docs/wcm.md
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
# nf-core/configs: Weill Cornell Medicine Configuration
|
||||||
|
|
||||||
|
All nf-core pipelines have been successfully configured for use on the panda cluster at the WCM.
|
||||||
|
|
||||||
|
To use, run the pipeline with `-profile wcm`. This will download and launch the [`wcm.config`](../conf/wcm.config) which has been pre-configured with a setup suitable for the WCM slurm cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline.
|
||||||
|
|
||||||
|
## Running the workflow on the Pasteur cluster
|
||||||
|
|
||||||
|
Nextflow is not installed by default on the WCM cluster.
|
||||||
|
|
||||||
|
- Install Nextflow : [here](https://www.nextflow.io/docs/latest/getstarted.html#)
|
||||||
|
|
||||||
|
Nextflow manages each process as a separate job that is submitted to the cluster by using the `sbatch` command.
|
||||||
|
Nextflow shouldn't run directly on a login node but on a compute node or lab-specific interactive server when configured as a submit host.
|
||||||
|
|
||||||
|
1. Run nextflow on a compute node or interactive server with submit host capability:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run nextflow workflow
|
||||||
|
nextflow run \\
|
||||||
|
nf-core/chipseq \\
|
||||||
|
-resume \\
|
||||||
|
-profile test,wcm
|
||||||
|
```
|
|
@ -25,6 +25,7 @@ profiles {
|
||||||
ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" }
|
ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" }
|
||||||
eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" }
|
eddie { includeConfig "${params.custom_config_base}/conf/eddie.config" }
|
||||||
icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" }
|
icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" }
|
||||||
|
ifb_core { includeConfig "${params.custom_config_base}/conf/ifb_core.config" }
|
||||||
imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" }
|
imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" }
|
||||||
imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" }
|
imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" }
|
||||||
genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" }
|
genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" }
|
||||||
|
@ -46,6 +47,7 @@ profiles {
|
||||||
uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" }
|
uppmax { includeConfig "${params.custom_config_base}/conf/uppmax.config" }
|
||||||
utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" }
|
utd_ganymede { includeConfig "${params.custom_config_base}/conf/utd_ganymede.config" }
|
||||||
uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" }
|
uzh { includeConfig "${params.custom_config_base}/conf/uzh.config" }
|
||||||
|
jax { includeConfig "${params.custom_config_base}/conf/jax.config" }
|
||||||
}
|
}
|
||||||
|
|
||||||
// If user hostnames contain one of these substring and they are
|
// If user hostnames contain one of these substring and they are
|
||||||
|
|
Loading…
Reference in a new issue