mirror of
https://github.com/MillironX/nf-configs.git
synced 2024-11-22 00:26:03 +00:00
Try new strategy for SGE/JAVA bs
This commit is contained in:
James Fellows Yates
parent
0c3428f5d1
commit
76fc461670
1 changed files with 224 additions and 12 deletions
|
|
@ -69,7 +69,7 @@ process {
|
|||
}
|
||||
|
||||
withName: adapter_removal {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 5)}G,virtual_free=${(task.memory.toGiga() * 5)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: dedup {
|
||||
|
|
@ -77,11 +77,11 @@ process {
|
|||
}
|
||||
|
||||
withName: markduplicates {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: library_merge {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: malt {
|
||||
|
|
@ -97,7 +97,7 @@ process {
|
|||
}
|
||||
|
||||
withName: mtnucratio {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 4)}G,virtual_free=${(task.memory.toGiga() * 4)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: vcf2genome {
|
||||
|
|
@ -105,11 +105,11 @@ process {
|
|||
}
|
||||
|
||||
withName: qualimap {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: damageprofiler {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() + 6)}G,virtual_free=${(task.memory.toGiga() + 6)}G" }
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 2)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: circularmapper {
|
||||
|
|
@ -129,6 +129,145 @@ process {
|
|||
|
||||
profiles {
|
||||
|
||||
medium_data {
|
||||
|
||||
params {
|
||||
// Specific nf-core/configs params
|
||||
config_profile_contact = 'James Fellows Yates (@jfy133)'
|
||||
config_profile_description = 'nf-core/eager medium-data EVA profile provided by nf-core/configs'
|
||||
}
|
||||
|
||||
executor {
|
||||
queueSize = 8
|
||||
}
|
||||
|
||||
process {
|
||||
|
||||
beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"'
|
||||
|
||||
maxRetries = 2
|
||||
|
||||
// Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
|
||||
clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" }
|
||||
|
||||
withLabel:'sc_tiny'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 1.5.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_small'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'sc_medium'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_small'{
|
||||
cpus = { check_max( 2, 'cpus' ) }
|
||||
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_medium' {
|
||||
cpus = { check_max( 4, 'cpus' ) }
|
||||
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_large'{
|
||||
cpus = { check_max( 8, 'cpus' ) }
|
||||
memory = { check_max( 24.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
withLabel:'mc_huge'{
|
||||
cpus = { check_max( 32, 'cpus' ) }
|
||||
memory = { check_max( 256.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
// Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
|
||||
|
||||
withName: makeSeqDict {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: fastqc {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: adapter_removal {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: dedup {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: markduplicates {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: library_merge {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: malt {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName:hostremoval_input_fastq {
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
|
||||
time = 1440.h
|
||||
}
|
||||
|
||||
withName: maltextract {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: multivcfanalyzer {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: mtnucratio {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: vcf2genome {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: qualimap {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: damageprofiler {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
}
|
||||
|
||||
withName: circularmapper {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: circulargenerator {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 2)}G" }
|
||||
}
|
||||
|
||||
withName: preseq {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 3)}G,virtual_free=${(task.memory.toGiga() * 3)}G" }
|
||||
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' }
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
big_data {
|
||||
|
||||
params {
|
||||
|
|
@ -143,13 +282,12 @@ profiles {
|
|||
|
||||
process {
|
||||
|
||||
beforeScript = 'export _JAVA_OPTIONS="-XX:ParallelGCThreads=1 -XX:+PrintCommandLineFlags"'
|
||||
|
||||
maxRetries = 2
|
||||
|
||||
withName:hostremoval_input_fastq {
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
|
||||
time = 1440.h
|
||||
}
|
||||
// Solution for clusterOptions comes from here: https://github.com/nextflow-io/nextflow/issues/332 + personal toMega conversion
|
||||
clusterOptions = { "-S /bin/bash -j y -o output.log -l h_vmem=${task.memory.toGiga()}G,virtual_free=${task.memory.toGiga()}G" }
|
||||
|
||||
withLabel:'sc_tiny'{
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
|
|
@ -192,6 +330,80 @@ profiles {
|
|||
memory = { check_max( 512.GB * task.attempt, 'memory' ) }
|
||||
time = { task.attempt == 3 ? 1440.h : task.attempt == 2 ? 48.h : 2.h }
|
||||
}
|
||||
|
||||
// Fixes for SGE and Java incompatibility due to Java using more memory than you tell it to use
|
||||
|
||||
withName: makeSeqDict {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: fastqc {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: adapter_removal {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: dedup {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: markduplicates {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: library_merge {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName:hostremoval_input_fastq {
|
||||
cpus = { check_max( 1, 'cpus' ) }
|
||||
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
|
||||
time = 1440.h
|
||||
}
|
||||
|
||||
withName: malt {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: maltextract {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: multivcfanalyzer {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: mtnucratio {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: vcf2genome {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: qualimap {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: damageprofiler {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: circularmapper {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: circulargenerator {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
}
|
||||
|
||||
withName: preseq {
|
||||
clusterOptions = { "-S /bin/bash -l h_vmem=${(task.memory.toGiga() * 6)}G,virtual_free=${(task.memory.toGiga() * 6)}G" }
|
||||
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'ignore' }
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue