Skip to content

Commit e13f5f8

Browse files
rsignellclaude
andcommitted
Add 60s timeout to thumbnail generation to prevent runner OOM kills
Large global datasets (e.g. ECMWF IFS ENS) can hang when loading chunks for thumbnail generation, causing the Actions runner to be killed. A SIGALRM-based timeout skips the thumbnail gracefully instead. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 95b534d commit e13f5f8

1 file changed

Lines changed: 25 additions & 1 deletion

File tree

scripts/build_dynamical_stac.py

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import logging
4141
import os
4242
import shutil
43+
import signal
4344
import subprocess
4445
import sys
4546
import tempfile
@@ -62,6 +63,21 @@
6263

6364
log = logging.getLogger(__name__)
6465

66+
# ---------------------------------------------------------------------------
67+
# Thumbnail timeout
68+
# ---------------------------------------------------------------------------
69+
70+
THUMBNAIL_TIMEOUT_SECONDS = 60
71+
72+
73+
class _ThumbnailTimeout(Exception):
74+
pass
75+
76+
77+
def _thumbnail_timeout_handler(signum, frame):
78+
raise _ThumbnailTimeout()
79+
80+
6581
# ---------------------------------------------------------------------------
6682
# Constants
6783
# ---------------------------------------------------------------------------
@@ -234,6 +250,8 @@ def generate_thumbnail(
234250
log.warning(" No temperature_2m in %s -- skipping thumbnail", item_id)
235251
return None
236252

253+
signal.signal(signal.SIGALRM, _thumbnail_timeout_handler)
254+
signal.alarm(THUMBNAIL_TIMEOUT_SECONDS)
237255
try:
238256
import cartopy.crs as ccrs
239257
import cartopy.feature as cfeature
@@ -348,13 +366,19 @@ def generate_thumbnail(
348366
log.info(" Thumbnail saved: %s", out_path)
349367
return out_path
350368

369+
except _ThumbnailTimeout:
370+
log.warning(" Thumbnail timed out after %ds for %s -- skipping",
371+
THUMBNAIL_TIMEOUT_SECONDS, item_id)
372+
return None
351373
except Exception as exc:
352374
log.warning(" Thumbnail generation failed for %s: %s", item_id, exc)
375+
return None
376+
finally:
377+
signal.alarm(0) # cancel any remaining alarm
353378
try:
354379
plt.close("all")
355380
except Exception:
356381
pass
357-
return None
358382

359383

360384
# ---------------------------------------------------------------------------

0 commit comments

Comments
 (0)