mirror of
https://github.com/MillironX/nf-configs.git
synced 2024-11-13 05:03:10 +00:00
Merge pull request #417 from Sage-Bionetworks-Workflows/bgrande/sage-improve-downloads
Improve reliability of file transfers with retries and reduced concurrency
This commit is contained in:
commit
57f63add02
2 changed files with 4 additions and 0 deletions
|
@ -29,9 +29,12 @@ aws {
|
|||
region = "us-east-1"
|
||||
client {
|
||||
uploadChunkSize = 209715200
|
||||
uploadMaxThreads = 4
|
||||
}
|
||||
batch {
|
||||
maxParallelTransfers = 1
|
||||
maxTransferAttempts = 5
|
||||
delayBetweenAttempts = '120 sec'
|
||||
}
|
||||
}
|
||||
executor {
|
||||
|
|
|
@ -8,6 +8,7 @@ This global configuration includes the following tweaks:
|
|||
- Enable retries by default when exit codes relate to insufficient memory
|
||||
- Allow pending jobs to finish if the number of retries are exhausted
|
||||
- Increase the amount of time allowed for file transfers
|
||||
- Improve reliability of file transfers with retries and reduced concurrency
|
||||
- Increase the default chunk size for multipart uploads to S3
|
||||
- Slow down job submission rate to avoid overwhelming any APIs
|
||||
- Define the `check_max()` function, which is missing in Sarek v2
|
||||
|
|
Loading…
Reference in a new issue