Delete SHH profiles due to decommissioning
James Fellows Yates
3 years ago
parent
5c9896b687
commit
59af7f0eee
@ -1,220 +0,0 @@
|
||||
// Profile config names for nf-core/configs
|
||||
|
||||
params {
|
||||
// Specific nf-core/configs params
|
||||
config_profile_contact = 'James Fellows Yates (@jfy133)'
|
||||
config_profile_description = 'nf-core/eager SHH profile provided by nf-core/configs'
|
||||
igenomes_base = "/projects1/public_data/igenomes/"
|
||||
}
|
||||
|
||||
// Specific nf-core/eager process configuration
|
||||
process {
|
||||
|
||||
maxRetries = 3
|
||||
|
||||
withName: malt {
|
||||
maxRetries = 1
|
||||
memory = { task.attempt > 1 ? 1900.GB : 725.GB }
|
||||
cpus = { task.attempt > 1 ? 112 : 64 }
|
||||
time = 1440.h
|
||||
queue = { task.memory > 756.GB ? 'supercruncher' : 'long' }
|
||||
}
|
||||
|
||||
withName: circulargenerator {
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
|
||||
time = { check_max( 4.h * task.attempt, 'time' ) }
|
||||
}
|
||||
|
||||
withLabel:'sc_tiny'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 1.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_small'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_medium'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_small'{
|
||||
cpus = { check_max( 2, 'cpus' ) }
|
||||
memory = { check_max( 4.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_medium' {
|
||||
cpus = { check_max( 4, 'cpus' ) }
|
||||
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_large'{
|
||||
cpus = { check_max( 8, 'cpus' ) }
|
||||
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_huge'{
|
||||
cpus = { check_max( 32, 'cpus' ) }
|
||||
memory = { check_max( 256.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
profiles {
|
||||
|
||||
big_data {
|
||||
|
||||
params {
|
||||
// Specific nf-core/configs params
|
||||
config_profile_contact = 'James Fellows Yates (@jfy133)'
|
||||
config_profile_description = 'nf-core/eager big-data SHH profile provided by nf-core/configs'
|
||||
}
|
||||
|
||||
executor {
|
||||
queueSize = 6
|
||||
}
|
||||
|
||||
process {
|
||||
|
||||
maxRetries = 2
|
||||
|
||||
withName:malt {
|
||||
maxRetries = 1
|
||||
memory = { task.attempt > 1 ? 1900.GB : 725.GB }
|
||||
cpus = { task.attempt > 1 ? 112 : 64 }
|
||||
time = 1440.h
|
||||
queue = { task.memory > 756.GB ? 'supercruncher' : 'long' }
|
||||
}
|
||||
|
||||
withName:hostremoval_input_fastq {
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
|
||||
time = 1440.h
|
||||
queue = 'long'
|
||||
}
|
||||
|
||||
withLabel:'sc_tiny'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 2.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_small'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_medium'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_small'{
|
||||
cpus = { check_max( 2, 'cpus' ) }
|
||||
memory = { check_max( 8.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_medium' {
|
||||
cpus = { check_max( 4, 'cpus' ) }
|
||||
memory = { check_max( 16.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_large'{
|
||||
cpus = { check_max( 8, 'cpus' ) }
|
||||
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_huge'{
|
||||
cpus = { check_max( 32, 'cpus' ) }
|
||||
memory = { check_max( 512.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : 48.h }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
microbiome_screening {
|
||||
process {
|
||||
// Assuming NCBI NT-sized DB
|
||||
withName: malt {
|
||||
maxRetries = 1
|
||||
memory = 1900.GB
|
||||
cpus = 112
|
||||
time = 1440.h
|
||||
queue = 'supercruncher'
|
||||
}
|
||||
}
|
||||
}
|
||||
// IMPORTANT this profile is not reproducible due to hardcoded paths. For initial/automated screening ONLY.
|
||||
hops {
|
||||
params {
|
||||
config_profile_description = 'Rough HOPS screening MPI-SHH profile, provided by nf-core/configs.'
|
||||
fasta = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta'
|
||||
bwa_index = '/projects1/Reference_Genomes/Human/HG19/'
|
||||
fasta_index = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.fasta.fai'
|
||||
seq_dict = '/projects1/Reference_Genomes/Human/HG19/hg19_complete.dict'
|
||||
bwaalnn = 0.01
|
||||
bwaalnl = 16
|
||||
run_bam_filtering = true
|
||||
bam_unmapped_type = 'fastq'
|
||||
run_metagenomic_screening = true
|
||||
metagenomic_tool = 'malt'
|
||||
metagenomic_min_support_reads = 1
|
||||
database = '/projects1/malt/databases/indexed/index040/full-bac-full-vir-etal-nov_2017'
|
||||
percent_identity = 85
|
||||
malt_mode = 'BlastN'
|
||||
malt_alignment_mode = 'SemiGlobal'
|
||||
malt_top_percent = 1
|
||||
malt_min_support_mode = 'reads'
|
||||
malt_max_queries = 100
|
||||
malt_memory_mode = 'load'
|
||||
run_maltextract = true
|
||||
maltextract_taxon_list = '/projects1/users/key/anc5h/soi.backup/List_of_pathogens_KB_fmk12_wViruses1.txt'
|
||||
maltextract_ncbifiles = '/projects1/clusterhomes/huebler/RMASifter/RMA_Extractor_Resources/'
|
||||
maltextract_filter = 'def_anc'
|
||||
maltextract_toppercent = 0.01
|
||||
maltextract_destackingoff = false
|
||||
maltextract_downsamplingoff = false
|
||||
maltextract_duplicateremovaloff = false
|
||||
maltextract_matches = false
|
||||
maltextract_megansummary = true
|
||||
maltextract_percentidentity = 85.0
|
||||
maltextract_topalignment = false
|
||||
}
|
||||
}
|
||||
pathogen_loose {
|
||||
params {
|
||||
config_profile_description = 'Pathogen (loose) MPI-SHH profile, provided by nf-core/configs.'
|
||||
bwaalnn = 0.01
|
||||
bwaalnl = 16
|
||||
}
|
||||
}
|
||||
pathogen_strict {
|
||||
params {
|
||||
config_profile_description = 'Pathogen (strict) MPI-SHH SDAG profile, provided by nf-core/configs.'
|
||||
bwaalnn = 0.1
|
||||
bwaalnl = 32
|
||||
}
|
||||
}
|
||||
human {
|
||||
params {
|
||||
config_profile_description = 'Human MPI-SHH SDAG profile, provided by nf-core/configs.'
|
||||
bwaalnn = 0.01
|
||||
bwaalnl = 16500
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
//Profile config names for nf-core/configs
|
||||
params {
|
||||
config_profile_description = 'Generic MPI-SHH SDAG cluster profile provided by nf-core/configs.'
|
||||
config_profile_contact = 'James Fellows Yates (@jfy133), Maxime Borry (@Maxibor)'
|
||||
config_profile_url = 'https://shh.mpg.de'
|
||||
max_memory = 2.TB
|
||||
max_cpus = 128
|
||||
max_time = 720.h
|
||||
//Illumina iGenomes reference file path
|
||||
igenomes_base = "/projects1/public_data/igenomes/"
|
||||
}
|
||||
|
||||
// Preform work directory cleanup after a successful run
|
||||
cleanup = true
|
||||
|
||||
process {
|
||||
executor = 'slurm'
|
||||
queue = { task.memory > 756.GB || task.cpus > 64 ? 'supercruncher': task.time <= 2.h ? 'short' : task.time <= 48.h ? 'medium': 'long' }
|
||||
}
|
||||
|
||||
executor {
|
||||
queueSize = 8
|
||||
}
|
||||
|
||||
profiles {
|
||||
singularity {
|
||||
singularity.enabled = true
|
||||
singularity.autoMounts = true
|
||||
singularity.cacheDir = "/projects1/singularity_scratch/cache/"
|
||||
}
|
||||
// Profile to deactivate automatic cleanup of work directory after a successful run. Overwrites cleanup option.
|
||||
debug {
|
||||
cleanup = false
|
||||
}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
# nf-core/configs: shh eager specific configuration
|
||||
|
||||
Extra specific configuration for eager pipeline
|
||||
|
||||
## Usage
|
||||
|
||||
To use, run the pipeline with `-profile shh`.
|
||||
|
||||
This will download and launch the eager specific [`shh.config`](../../../conf/pipeline/eager/shh.config) which has been pre-configured with a setup suitable for the shh cluster.
|
||||
|
||||
Example: `nextflow run nf-core/eager -profile shh`
|
||||
|
||||
## eager specific configurations for shh
|
||||
|
||||
Specific configurations for shh has been made for eager.
|
||||
|
||||
### General profiles
|
||||
|
||||
* If running with the MALT module turned on, the MALT process by default will be sent to the long queue with a resource requirement minimum of 725GB and 64 cores. If this fails, the process will be tried once more only and sent to the supercruncher queue. The module will not retry after this, and pipeline will fail. Note, this will only work on SDAG.
|
||||
|
||||
### Contextual profiles
|
||||
|
||||
#### Microbiome Sciences
|
||||
|
||||
* `microbiome_screening` runs MALT straight to supercruncher (with no retries!) and full resources requested due to microbiome screening databases often easily reach this size
|
||||
|
||||
#### Human Pop-Gen
|
||||
|
||||
* `human`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16500, -n 0.01`)
|
||||
|
||||
#### Pathogen
|
||||
|
||||
* `pathogen_loose`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 16 -n 0.01`)
|
||||
* `pathogen_strict`: optimised for mapping of human aDNA reads (i.e. bwa aln defaults as `-l 32, -n 0.1`)
|
||||
* `hops`: profile with default paths and parameters for automated/initial pathogen screening.
|
||||
* :warning: This is NOT a reproducible profile as it contains hardcoded paths. This should only be used for initial/automated screening where you wish to quickly check for any possible positives; after which you should re-do screening in a reproducible manner for publication!
|
||||
@ -1,16 +0,0 @@
|
||||
# nf-core/configs: SHH Configuration
|
||||
|
||||
All nf-core pipelines have been successfully configured for use on the Department of Archaeogenetic's SDAG/CDAG clusters at the [Max Planck Institute for the Science of Human History (MPI-SHH)](http://shh.mpg.de).
|
||||
|
||||
To use, run the pipeline with `-profile shh`. You can further with optimise submissions by specifying which cluster you are using with `-profile shh,sdag` or `-profile shh,cdag`. This will download and launch the [`shh.config`](../conf/shh.config) which has been pre-configured with a setup suitable for the SDAG and CDAG clusters respectively. Using this profile, a docker image containing all of the required software will be downloaded, and converted to a Singularity image before execution of the pipeline. The image will currently be centrally stored here:
|
||||
|
||||
```bash
|
||||
/projects1/singularity_scratch/cache/
|
||||
```
|
||||
|
||||
however this will likely change to a read-only directory in the future that will be managed by the IT team.
|
||||
|
||||
This configuration will automatically choose the correct SLURM queue (`short`,`medium`,`long`) depending on the time and memory required by each process. `-profile shh,sdag` additionally allows for submission of jobs to the `supercruncher` queue when a job's requested memory exceeds 756GB.
|
||||
|
||||
>NB: You will need an account and VPN access to use the cluster at MPI-SHH in order to run the pipeline. If in doubt contact the IT team.
|
||||
>NB: Nextflow will need to submit the jobs via SLURM to the clusters and as such the commands above will have to be executed on one of the head nodes. If in doubt contact IT.
|
||||
@ -1 +1 @@
|
||||
includeConfig("nfcore_custom.config")
|
||||
includeConfig("nfcore_custom.config")
|
||||
|
||||
Loading…
Reference in New Issue