From f5c36dde3ee04c89381f113c13b39c6fe64fe367 Mon Sep 17 00:00:00 2001 From: phue Date: Wed, 16 Jun 2021 16:36:39 +0200 Subject: [PATCH] cbe: send SIGUSR2 upon job termination Previously, if a process hit the walltime limit and received SIGKILL from the slurm scheduler, singularity did not properly propagate such (soft) kill signal. This prevented the exit code to be caught, e.g for resubmission purposes. This commit introduces a workaround using slurms --signal directive to send SIGUSR2 to the singularity process itself (instead of container child processes, which presumably was happening before). Effectively, once a job reaches walltime limit, this will result in exitcode 140 which is typically caught by the errorStrategy in nf-core pipelines See also: https://slurm.schedmd.com/sbatch.html#OPT_signal https://github.com/nextflow-io/nextflow/issues/2163 https://github.com/nextflow-io/nextflow/issues/1561 --- conf/cbe.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/cbe.config b/conf/cbe.config index 18f72dc..2ff0b6b 100755 --- a/conf/cbe.config +++ b/conf/cbe.config @@ -8,7 +8,7 @@ params { process { executor = 'slurm' queue = { task.memory <= 170.GB ? 'c' : 'm' } - clusterOptions = { task.time <= 1.h ? '--qos rapid' : task.time <= 8.h ? '--qos short': task.time <= 48.h ? '--qos medium' : '--qos long' } + clusterOptions = { def qos = task.time <= 1.h ? '--qos rapid' : { task.time <= 8.h ? '--qos short': { task.time <= 48.h ? '--qos medium' : '--qos long' } }; qos << ' --signal B:USR2' } module = 'anaconda3/2019.10' }