diff --git a/conf/vsc_ugent.config b/conf/vsc_ugent.config index b46b347..4a3733a 100644 --- a/conf/vsc_ugent.config +++ b/conf/vsc_ugent.config @@ -7,13 +7,19 @@ workDir = "$scratch_dir/work" // Perform work directory cleanup when the run has succesfully completed // cleanup = true -// Reduce the job submit rate to about 3 per second, this way the server won't be bombarded with jobs +// Reduce the job submit rate to about 30 per minute, this way the server won't be bombarded with jobs // Limit queueSize to keep job rate under control and avoid timeouts executor { - submitRateLimit = '3 sec' + submitRateLimit = '30/1min' queueSize = 50 } +// Add backoff strategy to catch cluster timeouts +process { + errorStrategy = { sleep(Math.pow(2, task.attempt) * 200 as long); return 'retry' } + maxRetries = 5 +} + // Specify that singularity should be used and where the cache dir will be for the images singularity { enabled = true @@ -30,7 +36,7 @@ profiles { skitty { params { config_profile_description = 'HPC_SKITTY profile for use on the Skitty cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 177.GB max_cpus = 36 @@ -40,7 +46,6 @@ profiles { process { executor = 'slurm' queue = 'skitty' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -48,7 +53,7 @@ profiles { swalot { params { config_profile_description = 'HPC_SWALOT profile for use on the Swalot cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 116.GB max_cpus = 20 @@ -58,7 +63,6 @@ profiles { process { executor = 'slurm' queue = 'swalot' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -66,7 +70,7 @@ profiles { victini { params { config_profile_description = 'HPC_VICTINI profile for use on the Victini cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 88.GB max_cpus = 36 @@ -76,7 +80,6 @@ profiles { process { executor = 'slurm' queue = 'victini' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -84,7 +87,7 @@ profiles { kirlia { params { config_profile_description = 'HPC_KIRLIA profile for use on the Kirlia cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 738.GB max_cpus = 36 @@ -94,7 +97,6 @@ profiles { process { executor = 'slurm' queue = 'kirlia' - maxRetries = 2 scratch = "$scratch_dir" } } @@ -102,7 +104,7 @@ profiles { doduo { params { config_profile_description = 'HPC_DODUO profile for use on the Doduo cluster of the VSC HPC.' - config_profile_contact = 'Nicolas Vannieuwkerke (@nvnieuwk)' + config_profile_contact = 'ict@cmgg.be' config_profile_url = 'https://www.ugent.be/hpc/en' max_memory = 250.GB max_cpus = 96 @@ -112,7 +114,6 @@ profiles { process { executor = 'slurm' queue = 'doduo' - maxRetries = 2 scratch = "$scratch_dir" } }