Reduce worker startup latency: fast env-activation entrypoint, lazy ML imports, slimmer bases

annotation_utilities/annotation_utilities/annotation_tools.py

@@ -1,6 +1,5 @@
 from shapely.geometry import Point, Polygon
 import numpy as np
-import matplotlib.colors as mcolors
 
 
 # Note that this function should reverse x and y. It is used by point_count_worker, which mixes up x and y for the polygons as well, so it works consistently.
@@ -298,6 +297,11 @@ def get_layers(GirderClient, datasetId):
 
 
 def process_and_merge_channels(images, layers, mode='lighten'):
+    # Imported lazily: matplotlib is heavy (~50ms) and this is the only function
+    # in annotation_tools that uses it, yet annotation_tools is imported at module
+    # load by nearly every worker. See todo/worker-startup-latency.md.
+    import matplotlib.colors as mcolors
+
     layers = sorted(layers, key=lambda x: x['channel'])
     processed_channels = []
 

build_all_property_and_annotation_workers.sh

@@ -37,7 +37,7 @@ docker build -f ./workers/annotations/connect_timelapse/$DOCKERFILE -t annotatio
 # docker build -f ./workers/annotations/connect_timelapse/Dockerfile_M1 -t annotations/connect_time_lapse:latest ./workers/annotations/connect_timelapse/
 
 echo "Building annotation worker: laplacian_of_gaussian"
-docker build -f ./workers/annotations/laplacian_of_gaussian/$DOCKERFILE -t annotations/laplacian_of_gaussian:latest . $NO_CACHE
+docker build -f ./workers/annotations/laplacian_of_gaussian/Dockerfile -t annotations/laplacian_of_gaussian:latest . $NO_CACHE
 # docker build -f ./workers/annotations/laplacian_of_gaussian/Dockerfile_M1 -t annotations/laplacian_of_gaussian:latest ./workers/annotations/laplacian_of_gaussian/
 
 
@@ -69,7 +69,7 @@ docker build -f ./workers/properties/blobs/blob_annulus_intensity_percentile_wor
 # docker build -f ./workers/properties/blobs/blob_annulus_intensity_percentile_worker/Dockerfile_M1 -t properties/blob_annulus_intensity_percentile:latest ./workers/properties/blobs/blob_annulus_intensity_percentile_worker/
 
 echo "Building property worker: blob_colony_two_color_intensity_worker"
-docker build -f ./workers/properties/blobs/blob_colony_two_color_intensity_worker/$DOCKERFILE -t properties/blob_colony_two_color_intensity:latest . $NO_CACHE
+docker build -f ./workers/properties/blobs/blob_colony_two_color_intensity_worker/Dockerfile -t properties/blob_colony_two_color_intensity:latest . $NO_CACHE
 # docker build -f ./workers/properties/blobs/blob_colony_two_color_intensity_worker/Dockerfile_M1 -t properties/blob_colony_two_color_intensity:latest ./workers/properties/blobs/blob_colony_two_color_intensity_worker/
 
 echo "Building property worker: blob_point_count_worker"
@@ -102,7 +102,7 @@ docker build -f ./workers/properties/points/point_to_nearest_blob_distance/$DOCK
 # docker build -f ./workers/properties/points/point_to_nearest_blob_distance/Dockerfile_M1 -t properties/point_to_nearest_blob_distance:latest ./workers/properties/points/point_to_nearest_blob_distance/
 
 echo "Building property worker: line_scan_worker"
-docker build -f ./workers/properties/lines/line_scan_worker/$DOCKERFILE -t properties/line_scan_worker:latest . $NO_CACHE
+docker build -f ./workers/properties/lines/line_scan_worker/Dockerfile -t properties/line_scan_worker:latest . $NO_CACHE
 # docker build -f ./workers/properties/lines/line_scan_worker/Dockerfile_M1 -t properties/line_scan_worker:latest ./workers/properties/lines/line_scan_worker/
 
 # Image processing workers

docker-compose.yml

@@ -48,7 +48,7 @@ services:
   laplacian_of_gaussian:
     build:
       context: .
-      dockerfile: ./workers/annotations/laplacian_of_gaussian/${DOCKERFILE:-Dockerfile}
+      dockerfile: ./workers/annotations/laplacian_of_gaussian/Dockerfile
     image: annotations/laplacian_of_gaussian:latest
     profiles: ["worker", "annotations"]
 
@@ -92,7 +92,7 @@ services:
   blob_colony_two_color_intensity:
     build:
       context: .
-      dockerfile: ./workers/properties/blobs/blob_colony_two_color_intensity_worker/${DOCKERFILE:-Dockerfile}
+      dockerfile: ./workers/properties/blobs/blob_colony_two_color_intensity_worker/Dockerfile
     image: properties/blob_colony_two_color_intensity:latest
     profiles: ["worker", "properties"]
 
@@ -156,7 +156,7 @@ services:
   line_scan_worker:
     build:
       context: .
-      dockerfile: ./workers/properties/lines/line_scan_worker/${DOCKERFILE:-Dockerfile}
+      dockerfile: ./workers/properties/lines/line_scan_worker/Dockerfile
     image: properties/line_scan_worker:latest
     profiles: ["worker", "properties"]
 

todo/TODO_REGISTRY.md

@@ -13,6 +13,7 @@ Master index of deferred work, technical debt, and future improvements for the I
 | ID | Title | Status | Priority | File | Related PR |
 |----|-------|--------|----------|------|------------|
 | TODO-001 | ML worker build optimization (mamba + shared base images) | Deferred | Medium | [ml-worker-build-optimization.md](ml-worker-build-optimization.md) | [#132](https://github.com/arjunrajlaboratory/ImageAnalysisProject/pull/132) |
+| TODO-002 | Worker startup latency audit & slimming plan (drop `conda run`) | In progress (#1–#3 done & verified; GPU workers static-validated, need build-host validation) | High | [worker-startup-latency.md](worker-startup-latency.md) | — |
 
 ## Completed TODOs