1
0
Fork 0
mirror of https://github.com/MillironX/nf-configs.git synced 2024-12-25 11:38:18 +00:00

Merge pull request #197 from combiz/imperial

Added institutional configs for Imperial and Imperial MEDBIO.  Added …
This commit is contained in:
Combiz Khozoie 2020-12-18 16:33:17 +00:00 committed by GitHub
commit ea2fd4be94
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 171 additions and 1 deletions

View file

@ -16,7 +16,7 @@ jobs:
needs: test_all_profiles
strategy:
matrix:
profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh']
profile: ['abims', 'awsbatch', 'bi','bigpurple', 'binac', 'cbe', 'ccga_dx', 'ccga_med', 'cfc', 'cfc_dev', 'crick', 'denbi_qbic', 'ebc', 'genotoul', 'genouest', 'gis', 'google', 'hebbe', 'icr_davros', 'imperial', 'imperial_mb', 'kraken', 'mpcdf', 'munin', 'pasteur', 'phoenix', 'prince', 'shh', 'uct_hpc', 'uppmax', 'utd_ganymede', 'uzh']
steps:
- uses: actions/checkout@v1
- name: Install Nextflow

37
conf/imperial.config Normal file
View file

@ -0,0 +1,37 @@
//Profile config names for nf-core/configs
params {
// Config Params
config_profile_description = 'Imperial College London - HPC Profile -- provided by nf-core/configs.'
config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)'
config_profile_url = 'https://www.imperial.ac.uk/admin-services/ict/self-service/research-support/rcs/'
// Resources
max_memory = 256.GB
max_cpus = 32
max_time = 72.h
}
executor {
$pbspro {
queueSize = 50
}
$local {
cpus = 2
queueSize = 1
memory = '32 GB'
}
}
singularity {
enabled = true
autoMounts = true
runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp"
}
process {
executor = 'pbspro'
}

44
conf/imperial_mb.config Normal file
View file

@ -0,0 +1,44 @@
//Profile config names for nf-core/configs
params {
// Config Params
config_profile_description = 'Imperial College London - MEDBIO QUEUE - HPC Profile -- provided by nf-core/configs.'
config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)'
config_profile_url = 'https://www.imperial.ac.uk/bioinformatics-data-science-group/resources/uk-med-bio/'
// Resources
max_memory = 640.GB
max_cpus = 32
max_time = 168.h
}
executor {
$pbspro {
queueSize = 50
}
$local {
cpus = 2
queueSize = 1
memory = '32 GB'
}
}
singularity {
enabled = true
autoMounts = true
runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp"
}
process {
executor = 'pbspro'
queue = 'pqmedbio-tput'
//queue = 'med-bio' //!! this is an alias and shouldn't be used
withLabel:process_large {
queue = 'pqmedbio-large'
}
}

View file

@ -0,0 +1,18 @@
// scflow/imperial specific profile config
params {
// Config Params
config_profile_description = 'Imperial College London - HPC - nf-core/scFlow Profile -- provided by nf-core/configs.'
config_profile_contact = 'Combiz Khozoie (c.khozoie@imperial.ac.uk)'
// Analysis Resource Params
ctd_folder = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/refs/ctd"
ensembl_mappings = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/Analyses/scFlowResources/src/ensembl-ids/ensembl_mappings.tsv"
}
singularity {
enabled = true
autoMounts = true
cacheDir = "/rds/general/user/$USER/projects/ukdrmultiomicsproject/live/.singularity-cache"
runOptions = "-B /rds/,/rdsgpfs/,/rds/general/user/$USER/ephemeral/tmp/:/tmp,/rds/general/user/$USER/ephemeral/tmp/:/var/tmp"
}

16
docs/imperial.md Normal file
View file

@ -0,0 +1,16 @@
# nf-core/configs: Imperial CX1 HPC Configuration
All nf-core pipelines have been successfully configured for use on the CX1 cluster at Imperial College London HPC.
To use, run the pipeline with `-profile imperial`. This will download and launch the [`imperial.config`](../conf/imperial.config) which has been pre-configured with a setup suitable for the CX1 cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline.
Before running the pipeline you will need to load Nextflow using the environment module system on the CX1 cluster. You can do this by issuing the commands below:
```bash
## Load Nextflow and Singularity environment modules
module load Nextflow
```
>NB: You will need an account to use the HPC cluster CX1 in order to run the pipeline. If in doubt contact IT.
>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT.
>NB: To submit jobs to the Imperial College MEDBIO cluster, use `-profile imperial_mb` instead.

16
docs/imperial_mb.md Normal file
View file

@ -0,0 +1,16 @@
# nf-core/configs: Imperial MEDBIO HPC Configuration
All nf-core pipelines have been successfully configured for use on the MEDBIO cluster at Imperial College London HPC.
To use, run the pipeline with `-profile imperial_mb`. This will download and launch the [`imperial_mb.config`](../conf/imperial_mb.config) which has been pre-configured with a setup suitable for the MEDBIO cluster. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline.
Before running the pipeline you will need to load Nextflow using the environment module system on the head node. You can do this by issuing the commands below:
```bash
## Load Nextflow and Singularity environment modules
module load Nextflow
```
>NB: You will need an account to use the HPC cluster MEDBIO in order to run the pipeline. Access to the MEDBIO queue is exclusive. If in doubt contact IT.
>NB: Nextflow will need to submit the jobs via the job scheduler to the HPC cluster and as such the commands above will have to be executed on one of the login nodes. If in doubt contact IT.
>NB: To submit jobs to the standard CX1 cluster at Imperial College, use `-profile imperial` instead.

View file

@ -0,0 +1,21 @@
# nf-core/configs: Imperial scflow Specific Configuration
Extra specific configuration for the scflow pipeline
## Usage
To use, run the pipeline with `-profile imperial` or `-profile imperial_mb`.
This will download and launch the scflow specific [`imperial.config`](../../../conf/pipeline/scflow/imperial.config) which has been pre-configured with a setup suitable for the Imperial HPC cluster.
Example: `nextflow run nf-core/scflow -profile imperial`
## scflow specific configurations for Imperial
Specific configurations for Imperial have been made for scflow.
* Singularity `enabled` and `autoMounts` set to `true`
* Singularity `cacheDir` path set to an RDS location
* Singularity `runOptions` path set to bind (`-B`) RDS paths with container paths.
* Params `ctd_folder` set to an RDS location.
* Parms `ensembl_mappings` set to an RDS location.

View file

@ -24,6 +24,8 @@ profiles {
czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" }
ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" }
icr_davros { includeConfig "${params.custom_config_base}/conf/icr_davros.config" }
imperial { includeConfig "${params.custom_config_base}/conf/imperial.config" }
imperial_mb { includeConfig "${params.custom_config_base}/conf/imperial_mb.config" }
genotoul { includeConfig "${params.custom_config_base}/conf/genotoul.config" }
google { includeConfig "${params.custom_config_base}/conf/google.config" }
denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" }
@ -54,6 +56,8 @@ params {
cfc: ['.hpc.uni-tuebingen.de'],
crick: ['.thecrick.org'],
icr_davros: ['.davros.compute.estate'],
imperial: ['.hpc.ic.ac.uk'],
imperial_mb: ['.hpc.ic.ac.uk'],
genotoul: ['.genologin1.toulouse.inra.fr', '.genologin2.toulouse.inra.fr'],
genouest: ['.genouest.org'],
uppmax: ['.uppmax.uu.se'],

14
pipeline/scflow.config Normal file
View file

@ -0,0 +1,14 @@
/*
* -------------------------------------------------
* nfcore/scflow custom profile Nextflow config file
* -------------------------------------------------
* Config options for custom environments.
* Cluster-specific config options should be saved
* in the conf/pipeline/scflow folder and imported
* under a profile name here.
*/
profiles {
imperial { includeConfig "${params.custom_config_base}/conf/pipeline/scflow/imperial.config" }
imperial_mb { includeConfig "${params.custom_config_base}/conf/pipeline/scflow/imperial.config" } // intended
}