mirror of
https://github.com/MillironX/nf-configs.git
synced 2024-11-21 16:16:04 +00:00
Merge branch 'master' into bgrande/sage-aws
This commit is contained in:
commit
80831358f9
11 changed files with 200 additions and 6 deletions
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
|
@ -30,6 +30,7 @@ jobs:
|
|||
matrix:
|
||||
profile:
|
||||
- "abims"
|
||||
- "adcra"
|
||||
- "alice"
|
||||
- "aws_tower"
|
||||
- "awsbatch"
|
||||
|
@ -49,6 +50,7 @@ jobs:
|
|||
- "cheaha"
|
||||
- "computerome"
|
||||
- "crick"
|
||||
- "crukmi"
|
||||
- "denbi_qbic"
|
||||
- "ebc"
|
||||
- "eddie"
|
||||
|
|
|
@ -67,6 +67,8 @@ Before adding your config file to nf-core/configs, we highly recommend writing a
|
|||
N.B. In your config file, please also make sure to add an extra `params` section with `params.config_profile_description`, `params.config_profile_contact` and `params.config_profile_url` set to reasonable values.
|
||||
Users will get information on who wrote the configuration profile then when executing a nf-core pipeline and can report back if there are things missing for example.
|
||||
|
||||
N.B. If you try to specify a shell environment variable within your profile, in some cases you may get an error during testing of something like `Unknown config attribute env.USER_SCRATCH -- check config file: /home/runner/work/configs/configs/nextflow.config` (where the bash environment variable is `$USER_SCRATCH`). This is because the github runner will not have your institutional environment variables set. To fix this you can define this as an internal variable, and set a fallback value for that variable. A good example is in the [VSC_UGENT profile](`https://github.com/nf-core/configs/blob/69468e7ca769643b151a6cfd1ab24185fc341c06/conf/vsc_ugent.config#L2`).
|
||||
|
||||
### Testing
|
||||
|
||||
If you want to add a new custom config file to `nf-core/configs` please test that your pipeline of choice runs as expected by using the [`-c`](https://www.nextflow.io/docs/latest/config.html) parameter.
|
||||
|
@ -86,6 +88,7 @@ See [`nf-core/configs/docs`](https://github.com/nf-core/configs/tree/master/docs
|
|||
Currently documentation is available for the following systems:
|
||||
|
||||
- [ABIMS](docs/abims.md)
|
||||
- [ADCRA](docs/adcra.md)
|
||||
- [ALICE](docs/alice.md)
|
||||
- [AWSBATCH](docs/awsbatch.md)
|
||||
- [AWS_TOWER](docs/aws_tower.md)
|
||||
|
@ -103,6 +106,7 @@ Currently documentation is available for the following systems:
|
|||
- [CHEAHA](docs/cheaha.md)
|
||||
- [Computerome](docs/computerome.md)
|
||||
- [CRICK](docs/crick.md)
|
||||
- [Cancer Research UK Manchester Institute](docs/crukmi.md)
|
||||
- [CZBIOHUB_AWS](docs/czbiohub.md)
|
||||
- [DENBI_QBIC](docs/denbi_qbic.md)
|
||||
- [EBC](docs/ebc.md)
|
||||
|
@ -203,6 +207,7 @@ Currently documentation is available for the following pipelines within specific
|
|||
- rnavar
|
||||
- [MUNIN](docs/pipeline/rnavar/munin.md)
|
||||
- sarek
|
||||
- [Cancer Research UK Manchester Institute](docs/pipeline/sarek/crukmi.md)
|
||||
- [MUNIN](docs/pipeline/sarek/munin.md)
|
||||
- [UPPMAX](docs/pipeline/sarek/uppmax.md)
|
||||
- taxprofiler
|
||||
|
|
40
conf/adcra.config
Normal file
40
conf/adcra.config
Normal file
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* --------------------------------------------------------------
|
||||
* nf-core pipelines config file for AD project using CRA HPC
|
||||
* --------------------------------------------------------------
|
||||
*/
|
||||
|
||||
params {
|
||||
config_profile_name = 'adcra'
|
||||
config_profile_description = 'CRA HPC profile provided by nf-core/configs'
|
||||
config_profile_contact = 'Kalayanee Chairat (@kalayaneech)'
|
||||
config_profile_url = 'https://bioinformatics.kmutt.ac.th/'
|
||||
}
|
||||
|
||||
params {
|
||||
max_cpus = 16
|
||||
max_memory = 128.GB
|
||||
max_time = 120.h
|
||||
}
|
||||
|
||||
// Specify the job scheduler
|
||||
executor {
|
||||
name = 'slurm'
|
||||
queueSize = 20
|
||||
submitRateLimit = '6/1min'
|
||||
}
|
||||
|
||||
singularity {
|
||||
enabled = true
|
||||
autoMounts = true
|
||||
}
|
||||
|
||||
process {
|
||||
scratch = true
|
||||
queue = 'unlimit'
|
||||
queueStatInterval = '10 min'
|
||||
maxRetries = 3
|
||||
errorStrategy = { task.attempt <=3 ? 'retry' : 'finish' }
|
||||
cache = 'lenient'
|
||||
exitStatusReadTimeoutMillis = '2700000'
|
||||
}
|
|
@ -1,3 +1,6 @@
|
|||
// Define the Scratch directory
|
||||
def scratch_dir = System.getenv("USER_SCRATCH") ?: "/tmp"
|
||||
|
||||
params {
|
||||
config_profile_name = 'cheaha'
|
||||
config_profile_description = 'University of Alabama at Birmingham Cheaha HPC'
|
||||
|
@ -6,14 +9,14 @@ params {
|
|||
}
|
||||
|
||||
env {
|
||||
TMPDIR="$USER_SCRATCH"
|
||||
SINGULARITY_TMPDIR="$USER_SCRATCH"
|
||||
TMPDIR="$USER"
|
||||
SINGULARITY_TMPDIR="$scratch_dir"
|
||||
}
|
||||
|
||||
singularity {
|
||||
enabled = true
|
||||
autoMounts = true
|
||||
runOptions = "--contain --workdir $USER_SCRATCH"
|
||||
runOptions = "--contain --workdir $scratch_dir"
|
||||
}
|
||||
|
||||
process {
|
||||
|
|
52
conf/crukmi.config
Normal file
52
conf/crukmi.config
Normal file
|
@ -0,0 +1,52 @@
|
|||
//Profile config names for nf-core/configs
|
||||
params {
|
||||
config_profile_description = 'Cancer Research UK Manchester Institute HPC cluster profile provided by nf-core/configs'
|
||||
config_profile_contact = 'Stephen Kitcatt, Simon Pearce (@skitcattCRUKMI, @sppearce)'
|
||||
config_profile_url = 'http://scicom.picr.man.ac.uk/projects/user-support/wiki'
|
||||
}
|
||||
|
||||
env {
|
||||
SINGULARITY_CACHEDIR = '/lmod/nextflow_software'
|
||||
}
|
||||
|
||||
singularity {
|
||||
enabled = true
|
||||
autoMounts = true
|
||||
}
|
||||
|
||||
process {
|
||||
beforeScript = 'module load apps/singularity/3.8.0'
|
||||
executor = 'pbs'
|
||||
|
||||
errorStrategy = {task.exitStatus in [143,137,104,134,139,140] ? 'retry' : 'finish'}
|
||||
maxErrors = '-1'
|
||||
maxRetries = 3
|
||||
|
||||
withLabel:process_low {
|
||||
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
|
||||
memory = { check_max( 5.GB * task.attempt, 'memory' ) }
|
||||
}
|
||||
|
||||
withLabel:process_medium {
|
||||
cpus = { check_max( 4 * task.attempt, 'cpus' ) }
|
||||
memory = { check_max( 20.GB * task.attempt, 'memory' ) }
|
||||
}
|
||||
|
||||
withLabel:process_high {
|
||||
cpus = { check_max( 16 * task.attempt, 'cpus' ) }
|
||||
memory = { check_max( 80.GB * task.attempt, 'memory' ) }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
executor {
|
||||
name = 'pbs'
|
||||
queueSize = 1000
|
||||
pollInterval = '10 sec'
|
||||
}
|
||||
|
||||
params {
|
||||
max_memory = 2000.GB
|
||||
max_cpus = 32
|
||||
max_time = 72.h
|
||||
}
|
18
conf/pipeline/sarek/crukmi.config
Normal file
18
conf/pipeline/sarek/crukmi.config
Normal file
|
@ -0,0 +1,18 @@
|
|||
// Profile config names for nf-core/configs
|
||||
|
||||
params {
|
||||
// Specific nf-core/configs params
|
||||
config_profile_description = 'Cancer Research UK Manchester Institute HPC cluster profile provided by nf-core/configs'
|
||||
config_profile_contact = 'Stephen Kitcatt, Simon Pearce (@skitcattCRUKMI, @sppearce)'
|
||||
config_profile_url = 'http://scicom.picr.man.ac.uk/projects/user-support/wiki'
|
||||
}
|
||||
|
||||
// Specific nf-core/sarek process configuration
|
||||
process {
|
||||
|
||||
withName: 'SAMTOOLS_MPILEUP' {
|
||||
cpus = 1
|
||||
memory = { 5.GB * task.attempt }
|
||||
}
|
||||
|
||||
}
|
39
docs/adcra.md
Normal file
39
docs/adcra.md
Normal file
|
@ -0,0 +1,39 @@
|
|||
# nf-core/configs: CRA HPC Configuration
|
||||
|
||||
nfcore pipeline sarek and rnaseq have been tested on the CRA HPC.
|
||||
|
||||
## Before running the pipeline
|
||||
|
||||
- You will need an account to use the CRA HPC cluster in order to run the pipeline.
|
||||
- Make sure that Singularity and Nextflow are installed.
|
||||
- Downlode pipeline singularity images to a HPC system using [nf-core tools](https://nf-co.re/tools/#downloading-pipelines-for-offline-use)
|
||||
|
||||
```
|
||||
$ conda install nf-core
|
||||
$ nf-core download
|
||||
```
|
||||
|
||||
- You will need to specify a Singularity cache directory in your ~./bashrc. This will store your container images in this cache directory without repeatedly downloading them every time you run a pipeline. Since space on home directory is limited, using lustre file system is recommended.
|
||||
|
||||
```
|
||||
export NXF_SINGULARITY_CACHEDIR = "/lustre/fs0/storage/yourCRAAccount/cache_dir"
|
||||
```
|
||||
|
||||
- Download iGenome reference to be used as a local copy.
|
||||
|
||||
```
|
||||
$ aws s3 --no-sign-request --region eu-west-1 sync s3://ngi-igenomes/igenomes/Homo_sapiens/GATK/GRCh38/ /lustre/fs0/storage/yourCRAAccount/references/Homo_sapiens/GATK/GRCh38/
|
||||
```
|
||||
|
||||
## Running the pipeline using the adcra config profile
|
||||
|
||||
- Run the pipeline within a [screen](https://linuxize.com/post/how-to-use-linux-screen/) or [tmux](https://linuxize.com/post/getting-started-with-tmux/) session.
|
||||
- Specify the config profile with `-profile adcra`.
|
||||
- Using lustre file systems to store results (`--outdir`) and intermediate files (`-work-dir`) is recommended.
|
||||
|
||||
```
|
||||
nextflow run /path/to/nf-core/<pipeline-name> -profile adcra \
|
||||
--genome GRCh38 \
|
||||
--igenomes_base /path/to/genome_references/ \
|
||||
... # the rest of pipeline flags
|
||||
```
|
15
docs/crukmi.md
Normal file
15
docs/crukmi.md
Normal file
|
@ -0,0 +1,15 @@
|
|||
# nf-core/configs: Cancer Research UK Manchester Institute Configuration
|
||||
|
||||
All nf-core pipelines have been successfully configured for the use on the HPC (phoenix) at Cancer Research UK Manchester Institute.
|
||||
|
||||
To use, run the pipeline with `-profile crukmi`. This will download and launch the [`crukmi.config`](../conf/crukmi.config) which has been pre-configured with a setup suitable for the phoenix HPC. Using this profile, singularity images will be downloaded to run on the cluster.
|
||||
|
||||
Before running the pipeline you will need to load Nextflow using the environment module system, for example via:
|
||||
|
||||
```bash
|
||||
## Load Nextflow and Singularity environment modules
|
||||
module purge
|
||||
module load apps/nextflow/22.04.5
|
||||
```
|
||||
|
||||
The pipeline should always be executed inside a workspace on the `/scratch/` system. All of the intermediate files required to run the pipeline will be stored in the `work/` directory. It is recommended to delete this directory after the pipeline has finished successfully because it can get quite large, and all of the main output files will be saved in the `results/` directory.
|
17
docs/pipeline/sarek/crukmi.md
Normal file
17
docs/pipeline/sarek/crukmi.md
Normal file
|
@ -0,0 +1,17 @@
|
|||
# nf-core/configs: CRUK-MI sarek specific configuration
|
||||
|
||||
Extra specific configuration for sarek pipeline
|
||||
|
||||
## Usage
|
||||
|
||||
To use, run the pipeline with `-profile crukmi`.
|
||||
|
||||
This will download and launch the sarek specific [`crukmi.config`](../../../conf/pipeline/sarek/munin.config) which has been pre-configured with a setup suitable for the Cancer Research UK Manchester Institute cluster (phoenix).
|
||||
|
||||
Example: `nextflow run nf-core/sarek -profile crukmi`
|
||||
|
||||
## Sarek specific configurations for CRUK-MI
|
||||
|
||||
Specific configurations for `CRUK-MI` has been made for sarek.
|
||||
|
||||
- Initial requested resources for SAMTOOLS_MPILEUP are only 5GB and 1 core.
|
|
@ -11,6 +11,7 @@
|
|||
//Please use a new line per include Config section to allow easier linting/parsing. Thank you.
|
||||
profiles {
|
||||
abims { includeConfig "${params.custom_config_base}/conf/abims.config" }
|
||||
adcra { includeConfig "${params.custom_config_base}/conf/adcra.config" }
|
||||
alice { includeConfig "${params.custom_config_base}/conf/alice.config" }
|
||||
aws_tower { includeConfig "${params.custom_config_base}/conf/aws_tower.config" }
|
||||
awsbatch { includeConfig "${params.custom_config_base}/conf/awsbatch.config" }
|
||||
|
@ -30,6 +31,7 @@ profiles {
|
|||
cheaha { includeConfig "${params.custom_config_base}/conf/cheaha.config" }
|
||||
computerome { includeConfig "${params.custom_config_base}/conf/computerome.config" }
|
||||
crick { includeConfig "${params.custom_config_base}/conf/crick.config" }
|
||||
crukmi { includeConfig "${params.custom_config_base}/conf/crukmi.config" }
|
||||
czbiohub_aws { includeConfig "${params.custom_config_base}/conf/czbiohub_aws.config" }
|
||||
denbi_qbic { includeConfig "${params.custom_config_base}/conf/denbi_qbic.config" }
|
||||
ebc { includeConfig "${params.custom_config_base}/conf/ebc.config" }
|
||||
|
|
|
@ -9,10 +9,11 @@
|
|||
*/
|
||||
|
||||
profiles {
|
||||
munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" }
|
||||
uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" }
|
||||
icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" }
|
||||
cfc { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" }
|
||||
cfc_dev { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/cfc.config" }
|
||||
crukmi { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/crukmi.config" }
|
||||
eddie { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/eddie.config" }
|
||||
icr_davros { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/icr_davros.config" }
|
||||
munin { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/munin.config" }
|
||||
uppmax { includeConfig "${params.custom_config_base}/conf/pipeline/sarek/uppmax.config" }
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue