Replace SIGALRM thumbnail timeout with subprocess hard-kill

SIGALRM doesn't reliably interrupt dask/zarr C-level I/O threads and
caused the GitHub Actions runner to crash. Running thumbnail generation
in a subprocess allows clean hard-killing via p.kill() after the timeout,
terminating all I/O threads cleanly without affecting the main process.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: rsignell

scripts/build_dynamical_stac.py

@@ -38,9 +38,9 @@
 
 import argparse
 import logging
+import multiprocessing
 import os
 import shutil
-import signal
 import subprocess
 import sys
 import tempfile
@@ -63,21 +63,9 @@
 
 log = logging.getLogger(__name__)
 
-# ---------------------------------------------------------------------------
-# Thumbnail timeout
-# ---------------------------------------------------------------------------
-
 THUMBNAIL_TIMEOUT_SECONDS = 60
 
 
-class _ThumbnailTimeout(Exception):
-    pass
-
-
-def _thumbnail_timeout_handler(signum, frame):
-    raise _ThumbnailTimeout()
-
-
 # ---------------------------------------------------------------------------
 # Constants
 # ---------------------------------------------------------------------------
@@ -232,56 +220,71 @@ def xarray_open_snippet(item_id: str, catalog_url: str) -> str:
 # Thumbnail generation
 # ---------------------------------------------------------------------------
 
-def generate_thumbnail(
-    session,
-    ds: "xr.Dataset",
+def _thumbnail_worker(
+    bucket: str,
+    prefix: str,
+    region: str,
     item_id: str,
-    output_dir: Path,
+    output_dir_str: str,
     temporal_dimension: str,
-) -> "Path | None":
-    """Generate a PNG temperature map thumbnail for a dataset.
-
-    Re-opens the store with dask chunks of size 1 on non-spatial dims so that
-    only the exact slice is fetched from S3, not the full multi-dim zarr chunk.
+    result_queue: "multiprocessing.Queue",
+) -> None:
+    """Subprocess worker: opens the icechunk store and generates a thumbnail PNG.
 
-    Returns the path to the saved PNG, or None if generation fails.
+    Runs in a separate process so it can be hard-killed on timeout without
+    leaving dask/zarr threads in a broken state in the main process.
     """
-    if "temperature_2m" not in ds:
-        log.warning("  No temperature_2m in %s -- skipping thumbnail", item_id)
-        return None
-
-    signal.signal(signal.SIGALRM, _thumbnail_timeout_handler)
-    signal.alarm(THUMBNAIL_TIMEOUT_SECONDS)
+    import warnings
+    import numpy as np
+    import matplotlib
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    import icechunk
+    import xarray as xr
+    from pathlib import Path
+
+    output_dir = Path(output_dir_str)
     try:
         import cartopy.crs as ccrs
         import cartopy.feature as cfeature
-        import numpy as np
 
-        # Re-open with dask using chunk size 1 on non-spatial dims.
-        # Without this, zarr loads the full stored chunk (which can span many
-        # lead_time / ensemble_member slices = many GB) just to extract one slice.
-        chunks = {d: 1 for d in ds.dims
-                  if d not in ("latitude", "longitude", "x", "y")}
+        storage = icechunk.s3_storage(
+            bucket=bucket, prefix=prefix, region=region, anonymous=True
+        )
+        repo = icechunk.Repository.open(storage=storage)
+        session = repo.readonly_session(branch="main")
+
+        # Open with chunk size 1 on non-spatial dims so zarr loads the minimum
+        # necessary data per slice (avoids pulling in huge stored chunks).
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="Numcodecs codecs are not in the Zarr version 3 specification.*",
+            )
+            ds = xr.open_zarr(session.store, chunks=None, consolidated=False, zarr_format=3)
+
+        if "temperature_2m" not in ds:
+            result_queue.put(None)
+            return
+
+        chunks = {d: 1 for d in ds.dims if d not in ("latitude", "longitude", "x", "y")}
         with warnings.catch_warnings():
             warnings.filterwarnings(
                 "ignore",
                 message="Numcodecs codecs are not in the Zarr version 3 specification.*",
             )
             ds_lazy = xr.open_zarr(
-                session.store, chunks=chunks,
-                consolidated=False, zarr_format=3
+                session.store, chunks=chunks, consolidated=False, zarr_format=3
             )
 
         da = ds_lazy["temperature_2m"]
-        log.info("  [thumb] dims=%s", dict(da.sizes))
 
-        # Select the most recent time slice
+        # Select a single time slice
         if temporal_dimension == "init_time" and "init_time" in da.dims:
             da = da.isel(init_time=-1, lead_time=0)
         elif temporal_dimension in da.dims:
             da = da.isel({temporal_dimension: -1})
 
-        # Pick one ensemble member to avoid loading all members
         if "ensemble_member" in da.dims:
             da = da.isel(ensemble_member=0)
 
@@ -297,88 +300,92 @@ def generate_thumbnail(
             da = da.isel(y=slice(None, None, y_step),
                          x=slice(None, None, x_step))
 
-        log.info("  [thumb] loading %s values...", dict(da.sizes))
-
-        # .compute() with dask fetches only the chunks we need
         data_c = da.compute().values - 273.15
-        log.info("  [thumb] loaded shape=%s  min=%.1f max=%.1f",
-                 data_c.shape, data_c.min(), data_c.max())
 
         is_projected = "x" in ds.dims and "y" in ds.dims
 
         if is_projected:
-            proj = ccrs.LambertConformal(
-                central_longitude=-97.5, central_latitude=38.5
-            )
-            fig, ax = plt.subplots(
-                figsize=(10, 6), subplot_kw={"projection": proj}
-            )
+            proj = ccrs.LambertConformal(central_longitude=-97.5, central_latitude=38.5)
+            fig, ax = plt.subplots(figsize=(10, 6), subplot_kw={"projection": proj})
             lons = da["longitude"].compute().values
             lats = da["latitude"].compute().values
-            img = ax.pcolormesh(
-                lons, lats, data_c,
-                cmap="RdBu_r", vmin=-40, vmax=40,
-                transform=ccrs.PlateCarree(),
-                shading="auto",
-            )
+            img = ax.pcolormesh(lons, lats, data_c, cmap="RdBu_r", vmin=-40, vmax=40,
+                                transform=ccrs.PlateCarree(), shading="auto")
             ax.add_feature(cfeature.COASTLINE, linewidth=0.5)
             ax.add_feature(cfeature.STATES, linewidth=0.3)
             ax.set_extent([-130, -60, 20, 55], crs=ccrs.PlateCarree())
         else:
             proj = ccrs.PlateCarree()
-            fig, ax = plt.subplots(
-                figsize=(10, 5), subplot_kw={"projection": proj}
-            )
-            # Determine lon/lat coordinate names
+            fig, ax = plt.subplots(figsize=(10, 5), subplot_kw={"projection": proj})
             lon_name = "longitude" if "longitude" in da.dims else "lon"
             lat_name = "latitude" if "latitude" in da.dims else "lat"
-            lons = da[lon_name].values
-            lats = da[lat_name].values
-            img = ax.pcolormesh(
-                lons, lats, data_c,
-                cmap="RdBu_r", vmin=-40, vmax=40,
-                transform=proj,
-                shading="auto",
-            )
+            img = ax.pcolormesh(da[lon_name].values, da[lat_name].values, data_c,
+                                cmap="RdBu_r", vmin=-40, vmax=40, transform=proj,
+                                shading="auto")
             ax.set_global()
             ax.add_feature(cfeature.COASTLINE, linewidth=0.5)
 
         plt.colorbar(img, ax=ax, orientation="horizontal", pad=0.04,
                      label="2m Temperature (°C)", shrink=0.7)
 
-        # Build timestamp string for title
         try:
             ts_val = da[temporal_dimension].values if temporal_dimension in da.coords else None
             ts_str = str(np.datetime_as_string(ts_val, unit="h")) if ts_val is not None else ""
         except Exception:
             ts_str = ""
 
-        title = item_id
-        if ts_str:
-            title += f"  |  {ts_str}"
-        ax.set_title(title, fontsize=9)
+        ax.set_title(f"{item_id}  |  {ts_str}" if ts_str else item_id, fontsize=9)
 
         thumb_dir = output_dir / "thumbnails"
         thumb_dir.mkdir(parents=True, exist_ok=True)
         out_path = thumb_dir / f"{item_id}.png"
         fig.savefig(out_path, dpi=100, bbox_inches="tight")
         plt.close(fig)
-        log.info("  Thumbnail saved: %s", out_path)
-        return out_path
+        result_queue.put(str(out_path))
+
+    except Exception as exc:
+        result_queue.put(None)
+        raise  # surfaces in the subprocess stderr for debugging
+
+
+def generate_thumbnail(
+    bucket: str,
+    prefix: str,
+    region: str,
+    item_id: str,
+    output_dir: Path,
+    temporal_dimension: str,
+) -> "Path | None":
+    """Generate a thumbnail PNG in a subprocess, hard-killing it after a timeout.
 
-    except _ThumbnailTimeout:
+    Using a subprocess (rather than a thread or SIGALRM) ensures that all
+    dask/zarr I/O threads are cleanly terminated if data loading stalls.
+    """
+    result_queue: multiprocessing.Queue = multiprocessing.Queue()
+    p = multiprocessing.Process(
+        target=_thumbnail_worker,
+        args=(bucket, prefix, region, item_id, str(output_dir),
+              temporal_dimension, result_queue),
+    )
+    p.start()
+    p.join(timeout=THUMBNAIL_TIMEOUT_SECONDS)
+
+    if p.is_alive():
         log.warning("  Thumbnail timed out after %ds for %s -- skipping",
                     THUMBNAIL_TIMEOUT_SECONDS, item_id)
+        p.kill()
+        p.join()
         return None
-    except Exception as exc:
-        log.warning("  Thumbnail generation failed for %s: %s", item_id, exc)
+
+    if p.exitcode != 0:
+        log.warning("  Thumbnail subprocess failed for %s (exit code %d)",
+                    item_id, p.exitcode)
         return None
-    finally:
-        signal.alarm(0)  # cancel any remaining alarm
-        try:
-            plt.close("all")
-        except Exception:
-            pass
+
+    if not result_queue.empty():
+        result = result_queue.get_nowait()
+        return Path(result) if result else None
+    return None
 
 
 # ---------------------------------------------------------------------------
@@ -446,7 +453,7 @@ def build_item_for_store(
     log.info("  Built item: %s  bbox=%s", item_id, item_dict["bbox"])
 
     if output_dir and thumbnail_url_base:
-        thumb_path = generate_thumbnail(session, ds, item_id, output_dir, temporal_dim)
+        thumb_path = generate_thumbnail(bucket, prefix, region, item_id, output_dir, temporal_dim)
         if thumb_path:
             item_dict["assets"]["thumbnail"] = {
                 "href": f"{thumbnail_url_base}/{item_id}.png",