From c27bd4ac6bd41664455801ae64fc46d3380e10c6 Mon Sep 17 00:00:00 2001 From: Rick Tankard Date: Wed, 8 Jan 2020 11:28:08 +0800 Subject: [PATCH] sbatch submission failures do not continue --- profiles/slurm/submit | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/profiles/slurm/submit b/profiles/slurm/submit index 0a35065..fef8ae7 100755 --- a/profiles/slurm/submit +++ b/profiles/slurm/submit @@ -50,6 +50,18 @@ fi export DEP_STRING export SBATCH_ARGS -# need to return just the job ID part to enable snakemake to build dependencies correctly -# $@ should still contain the job execution script generated by snakemake -sbatch ${DEP_STRING} ${SBATCH_ARGS} $@ | cut -f4 -d' ' +# Submit the job, retrying if sbatch fails to submit the job. +# It is important to not continue after a failure otherwise the JOBID is reported as an empty string. +JOBID='' +while true +do + # need to return just the job ID part to enable snakemake to build dependencies correctly + # $@ should still contain the job execution script generated by snakemake + JOBID=$(sbatch ${DEP_STRING} ${SBATCH_ARGS} $@ | cut -f4 -d' ') + if [ "$JOBID" != '' ]; then + break + fi + echo "Failed to submit job to slurm, waiting 10 seconds" 1>&2 + sleep 10 +done +echo -n $JOBID