diff --git a/conf/sage.config b/conf/sage.config index 4692ed2..c77f8fa 100644 --- a/conf/sage.config +++ b/conf/sage.config @@ -29,9 +29,12 @@ aws { region = "us-east-1" client { uploadChunkSize = 209715200 + uploadMaxThreads = 4 } batch { maxParallelTransfers = 1 + maxTransferAttempts = 5 + delayBetweenAttempts = '120 sec' } } executor { diff --git a/docs/sage.md b/docs/sage.md index 1e36fed..7b4a8fe 100644 --- a/docs/sage.md +++ b/docs/sage.md @@ -8,6 +8,7 @@ This global configuration includes the following tweaks: - Enable retries by default when exit codes relate to insufficient memory - Allow pending jobs to finish if the number of retries are exhausted - Increase the amount of time allowed for file transfers +- Improve reliability of file transfers with retries and reduced concurrency - Increase the default chunk size for multipart uploads to S3 - Slow down job submission rate to avoid overwhelming any APIs - Define the `check_max()` function, which is missing in Sarek v2