millironx/nf-configs
1
0
Fork 0
mirror of https://github.com/MillironX/nf-configs.git synced 2024-11-22 00:26:03 +00:00
Code Issues Releases Packages Activity

Try new strategy for SGE/JAVA bs

Browse source
This commit is contained in:
James Fellows Yates 2021-08-04 10:14:56 +02:00
parent 0c3428f5d1
commit 76fc461670
1 changed files with 224 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

236
conf/pipeline/eager/eva.config
View file

@ -69,7 +69,7 @@ process {
} }
withName: adapter_removal { withName: adapter_removal {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 5)}G,virtual_free=${(task.memory.toGiga() * 5)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: dedup { withName: dedup {
@ -77,11 +77,11 @@ process {
} }
withName: markduplicates { withName: markduplicates {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: library_merge { withName: library_merge {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: malt { withName: malt {
@ -97,7 +97,7 @@ process {
} }
withName: mtnucratio { withName: mtnucratio {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 4)}G,virtual_free=${(task.memory.toGiga() * 4)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: vcf2genome { withName: vcf2genome {
@ -105,11 +105,11 @@ process {
} }
withName: qualimap { withName: qualimap {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: damageprofiler { withName: damageprofiler {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" } clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
} }
withName: circularmapper { withName: circularmapper {
@ -129,6 +129,145 @@ process {
profiles { profiles {
medium_data {
params {
// Specific nf-core/configs params
config_profile_contact = 'James Fellows Yates (@jfy133)'
config_profile_description = 'nf-core/eager medium-data EVA profile provided by nf-core/configs'
}
executor {
queueSize = 8
}
process {
beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"'
maxRetries = 2
// Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" }
withLabel:'sc_tiny'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 1.5.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'sc_small'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'sc_medium'{
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'mc_small'{
cpus = { check_max( 2, 'cpus' ) }
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'mc_medium' {
cpus = { check_max( 4, 'cpus' ) }
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'mc_large'{
cpus = { check_max( 8, 'cpus' ) }
memory = { check_max( 24.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
withLabel:'mc_huge'{
cpus = { check_max( 32, 'cpus' ) }
memory = { check_max( 256.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
}
// Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
withName: makeSeqDict {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: fastqc {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: adapter_removal {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: dedup {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: markduplicates {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: library_merge {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: malt {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName:hostremoval_input_fastq {
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
time = 1440.h
}
withName: maltextract {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: multivcfanalyzer {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: mtnucratio {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: vcf2genome {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: qualimap {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: damageprofiler {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
}
withName: circularmapper {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
}
withName: circulargenerator {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
}
withName: preseq {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' }
}
}
}
big_data { big_data {
params { params {
@ -143,13 +282,12 @@ profiles {
process { process {
maxRetries = 2 beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"'
withName:hostremoval_input_fastq { maxRetries = 2
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 32.GB * task.attempt, 'memory' ) } // Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
time = 1440.h clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" }
}
withLabel:'sc_tiny'{ withLabel:'sc_tiny'{
cpus = { check_max( 1, 'cpus' ) } cpus = { check_max( 1, 'cpus' ) }
@ -192,6 +330,80 @@ profiles {
memory = { check_max( 512.GB * task.attempt, 'memory' ) } memory = { check_max( 512.GB * task.attempt, 'memory' ) }
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h } time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
} }
// Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
withName: makeSeqDict {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: fastqc {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: adapter_removal {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: dedup {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: markduplicates {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: library_merge {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName:hostremoval_input_fastq {
cpus = { check_max( 1, 'cpus' ) }
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
time = 1440.h
}
withName: malt {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: maltextract {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: multivcfanalyzer {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: mtnucratio {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: vcf2genome {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: qualimap {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: damageprofiler {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: circularmapper {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: circulargenerator {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
}
withName: preseq {
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' }
}
} }
} }