millironx/nf-configs
1
0
Fork 0
mirror of https://github.com/MillironX/nf-configs.git synced 2024-09-20 21:52:04 +00:00
Code Issues Releases Packages Activity

Simplify resource adjustments

Browse source
This commit is contained in:
Bruno Grande 2022-08-31 09:10:30 -07:00
parent 9c8cb71bc7
commit c883723559
2 changed files with 36 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
conf/sage.config
View file

@ -37,58 +37,43 @@ executor {
submitRateLimit = '5 / 1 sec'
}
// Disabling resource allocation tweaks for now
//
// params {
// max_memory = 500.GB
// max_cpus = 64
// max_time = 168.h // One week
// }
//
// process {
//
// cpus = { check_max( 1 * slow(task.attempt), 'cpus' ) }
// memory = { check_max( 6.GB * task.attempt, 'memory' ) }
// time = { check_max( 24.h * task.attempt, 'time' ) }
//
// // Process-specific resource requirements
// withLabel:process_low {
// cpus = { check_max( 4 * slow(task.attempt), 'cpus' ) }
// memory = { check_max( 12.GB * task.attempt, 'memory' ) }
// time = { check_max( 24.h * task.attempt, 'time' ) }
// }
// withLabel:process_medium {
// cpus = { check_max( 12 * slow(task.attempt), 'cpus' ) }
// memory = { check_max( 36.GB * task.attempt, 'memory' ) }
// time = { check_max( 48.h * task.attempt, 'time' ) }
// }
// withLabel:process_high {
// cpus = { check_max( 24 * slow(task.attempt), 'cpus' ) }
// memory = { check_max( 72.GB * task.attempt, 'memory' ) }
// time = { check_max( 96.h * task.attempt, 'time' ) }
// }
// withLabel:process_long {
// time = { check_max( 192.h * task.attempt, 'time' ) }
// }
// withLabel:process_high_memory {
// memory = { check_max( 128.GB * task.attempt, 'memory' ) }
// }
//
// // Preventing Sarek labels from using the actual maximums
// withLabel:memory_max {
// memory = { check_max( 128.GB * task.attempt, 'memory' ) }
// }
// withLabel:cpus_max {
// cpus = { check_max( 24 * slow(task.attempt), 'cpus' ) }
// }
//
// }
// Adjust default resource allocations (see `../docs/sage.md`)
process {
cpus = { check_max( 1 * slow(task.attempt), 'cpus' ) }
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
time = { check_max( 24.h * task.attempt, 'time' ) }
// Process-specific resource requirements
withLabel:process_low {
cpus = { check_max( 4 * slow(task.attempt), 'cpus' ) }
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
time = { check_max( 24.h * task.attempt, 'time' ) }
}
withLabel:process_medium {
cpus = { check_max( 12 * slow(task.attempt), 'cpus' ) }
memory = { check_max( 36.GB * task.attempt, 'memory' ) }
time = { check_max( 48.h * task.attempt, 'time' ) }
}
withLabel:process_high {
cpus = { check_max( 24 * slow(task.attempt), 'cpus' ) }
memory = { check_max( 72.GB * task.attempt, 'memory' ) }
time = { check_max( 96.h * task.attempt, 'time' ) }
}
withLabel:process_long {
time = { check_max( 192.h * task.attempt, 'time' ) }
}
withLabel:process_high_memory {
memory = { check_max( 128.GB * task.attempt, 'memory' ) }
}
}
// Function to slow the increase of the resource multipler
// as attempts are made. The rationale is that some CPUs
// don't need to be increased as fast as memory.
// as attempts are made. The rationale is that the number
// of CPU cores isn't a limiting factor as often as memory.
def slow(attempt, factor = 2) {
return Math.ceil( attempt / factor) as int
return Math.ceil( attempt / factor) as int
}

4
docs/sage.md
View file

@ -11,8 +11,8 @@ This global configuration includes the following tweaks:
- Increase the default chunk size for multipart uploads to S3
- Slow down job submission rate to avoid overwhelming any APIs
- Define the `check_max()` function, which is missing in Sarek v2
- (Disabled temporarily) Slow the increase in the number of allocated CPU cores on retries
- (Disabled temporarily) Increase the default time limits because we run pipelines on AWS
- Slow the increase in the number of allocated CPU cores on retries
- Increase the default time limits because we run pipelines on AWS
## Additional information about iGenomes