diff --git a/cli/USER_GUIDE.md b/cli/USER_GUIDE.md index 060bd63..2ba5d28 100644 --- a/cli/USER_GUIDE.md +++ b/cli/USER_GUIDE.md @@ -24,6 +24,8 @@ - [First-Time Setup](#first-time-setup) - [Day-to-Day Usage](#day-to-day-usage) - [CI/CD Comparison](#cicd-comparison) +- [Testing](#testing) + - [Testing Docker Image Management](#testing-docker-image-management) - [Troubleshooting](#troubleshooting) --- @@ -934,6 +936,130 @@ localci run --matrix compiler=clang --matrix version=20 --- +## Testing + +### Testing Docker Image Management + +How to test the Docker image management feature (registry, two-mark matching, +load/build/save, base images). + +#### Unit tests (no Docker required) + +Most tests mock Docker. From the **cli** directory: + +```bash +cd cli +pip install -e ".[dev]" # if not already +python3 -m pytest tests/test_registry.py tests/test_image_manager.py -v +``` + +**What they cover:** + +- **`tests/test_registry.py`**: Two-mark algorithm (essential/extra marks), + `select_image`, tie-breaking, `ImageRegistry` load/save/CRUD, queue builder + with registry (full match vs needs_build). +- **`tests/test_image_manager.py`**: `image_name_from_entry`, + `image_name_base_from_entry`, `ImageManager.prepare_image_for_job` (use + existing, load from tar) with mocked Docker. + +Run all related tests (including executor/orchestrator that use cache paths): + +```bash +python3 -m pytest tests/test_registry.py tests/test_image_manager.py tests/test_orchestrator.py tests/test_executor.py -v +``` + +#### CLI checks (list / info) + +With an `image-registry.yml` in the project (or use `--registry`), you can +test list and info without Docker: + +```bash +# Create a minimal registry so list works +echo 'version: "1.0" +images: + - name: capy-ubuntu-25.04-gcc15 + docker_tag: capy-ubuntu-25.04-gcc15:latest + file: images/capy/capy-ubuntu-25.04-gcc15.tar + os: ubuntu:25.04 + architecture: x86_64 + compilers: [gcc-15]' > image-registry.yml + +localci images list +localci images info capy-ubuntu-25.04-gcc15 +``` + +With Docker installed you can pass the registry path explicitly: + +```bash +localci images list --registry image-registry.yml +localci images info capy-ubuntu-25.04-gcc15 --registry image-registry.yml +``` + +#### End-to-end with Docker and a workflow + +**Prerequisites:** Docker running, `act` installed, and a workflow that uses a +matrix with `container:` (e.g. capy’s CI). + +**Registry present, image in registry** + +1. Create `image-registry.yml` in the project root (or where you run `localci`): + + ```yaml + version: "1.0" + images: + - name: capy-ubuntu-25.04-gcc15 + file: images/capy/capy-ubuntu-25.04-gcc15.tar + docker_tag: capy-ubuntu-25.04-gcc15:latest + os: ubuntu:25.04 + architecture: x86_64 + compilers: [gcc-15] + ``` + +2. Build or import the image and save as `.tar`: + + ```bash + docker build -t capy-ubuntu-25.04-gcc15:latest -f images/capy/Dockerfile.capy-ubuntu-25.04-gcc15 images/capy + mkdir -p images/capy + docker save -o images/capy/capy-ubuntu-25.04-gcc15.tar capy-ubuntu-25.04-gcc15:latest + ``` + +3. Run one job; the orchestrator should load the image from the registry: + + ```bash + cd /path/to/capy # or project with .github/workflows/ci.yml + localci run --workflow .github/workflows/ci.yml --job build --matrix compiler=gcc --matrix version=15 + ``` + +**No registry:** Without an `image-registry.yml` (or with an empty registry), +the queue uses base-only tags and does not build; `act` uses the default +runner image or you must provide images another way. + +**Registry present, no matching image (needs_build):** Use a registry with no +image matching your matrix (e.g. empty `images:` or different OS/compiler). +Run the same `localci run` as above. The queue sets `needs_build=True` and the +orchestrator calls `ImageManager.prepare_image_for_job`, which will try to +build a base image and save it if Docker and a suitable Dockerfile or +generated build path exist. + +#### Quick checklist + +| Test | Command / action | +|------|-------------------| +| Registry + matching | `pytest tests/test_registry.py -v` | +| Image manager (mocked) | `pytest tests/test_image_manager.py -v` | +| Queue builder + registry | In `test_registry.py`: `TestQueueBuilderWithRegistry` | +| Base-only naming | `pytest tests/test_image_manager.py::TestImageNameBaseFromEntry -v` | +| CLI list/info | `localci images list`, `localci images info ` with a valid `image-registry.yml` | +| Load from .tar | Build image, `docker save` to path in registry `file`, run `localci run`; check logs for “Loading image” / “Image ready”. | +| Build when no match | Empty or non-matching registry + `localci run`; confirm build and save. | + +**File reference:** Registry and matching: `localci/core/registry.py`. Image +manager (load/build/save): `localci/core/image_manager.py`. Queue image +resolution: `localci/core/queue_builder.py`. CLI: `localci/cli/images.py` +(`localci images list|info|build|clean|import|export`). + +--- + ## Troubleshooting ### "No config file found" diff --git a/cli/localci/cli/images.py b/cli/localci/cli/images.py index cd50dda..ef8729f 100644 --- a/cli/localci/cli/images.py +++ b/cli/localci/cli/images.py @@ -7,13 +7,16 @@ from __future__ import annotations import json +import re import subprocess +from datetime import datetime, timezone, timedelta from pathlib import Path import click import yaml from localci.core.registry import ImageRegistry +from localci.utils.docker import DockerManager from localci.utils.output import ( console, make_table, @@ -189,11 +192,28 @@ def images_build( # --------------------------------------------------------------------------- +def _parse_older_than(s: str) -> timedelta | None: + """Parse --older-than value: e.g. 7d, 30d, 2w, 1m (m = 30 days).""" + m = re.match(r"^(\d+)(d|w|m)$", s.strip().lower()) + if not m: + return None + num = int(m.group(1)) + unit = m.group(2) + if unit == "d": + return timedelta(days=num) + if unit == "w": + return timedelta(weeks=num) + if unit == "m": + return timedelta(days=num * 30) + return None + + @images.command("clean") -@click.option("--older-than", type=str, default=None, help="Remove images older than (e.g. 30d).") -@click.option("--unused", is_flag=True, help="Remove unused images.") -@click.option("--all", "clean_all", is_flag=True, help="Remove all localci images.") +@click.option("--older-than", type=str, default=None, help="Remove registry images not used since (e.g. 30d, 7d, 2w).") +@click.option("--unused", is_flag=True, help="Remove registry images with usage_count 0.") +@click.option("--all", "clean_all", is_flag=True, help="Remove all localci capy images (Docker + registry).") @click.option("--dry-run", is_flag=True, help="Preview without removing.") +@click.option("--registry", "-r", "registry_path", type=click.Path(path_type=Path, exists=False), default=None, help="Path to image-registry.yml.") @click.pass_context def images_clean( ctx: click.Context, @@ -201,17 +221,70 @@ def images_clean( unused: bool, clean_all: bool, dry_run: bool, + registry_path: Path | None, ) -> None: - """Clean up Docker images.""" - if older_than: - print_warning("--older-than is not yet implemented; ignoring.") - if unused: - print_warning("--unused is not yet implemented; ignoring.") + """Clean up Docker images and optionally registry / .tar files.""" + reg_path = registry_path or REGISTRY_FILE + project_dir = REPO_ROOT + + # Disk space management: --older-than and --unused (registry-based) + if older_than or unused: + if not reg_path.exists(): + print_error(f"Registry not found: {reg_path}. Cannot use --older-than/--unused.") + ctx.exit(1) + delta = None + if older_than: + delta = _parse_older_than(older_than) + if not delta: + print_error("--older-than must be like 7d, 30d, 2w, 1m") + ctx.exit(1) + registry = ImageRegistry(reg_path) + registry.load() + cutoff = (datetime.now(timezone.utc) - delta) if delta else None + to_remove: list[str] = [] + for e in registry.entries: + if older_than and cutoff and e.last_used: + try: + lu = datetime.fromisoformat(e.last_used.replace("Z", "+00:00")) + if lu.tzinfo is None: + lu = lu.replace(tzinfo=timezone.utc) + if lu < cutoff: + to_remove.append(e.name) + except ValueError: + pass + if unused and (e.usage_count or 0) == 0: + to_remove.append(e.name) + to_remove = list(dict.fromkeys(to_remove)) + if not to_remove: + print_info("No images match --older-than/--unused.") + return + docker = DockerManager() + for name in to_remove: + entry = registry.find_by_name(name) + if not entry: + continue + tag = entry.docker_tag + if dry_run: + console.print(f" Would remove: {name} (Docker: {tag})") + continue + if docker.image_exists(tag): + docker.remove_image(tag, force=True) + tar_path = project_dir / entry.file if not Path(entry.file).is_absolute() else Path(entry.file) + if tar_path.exists(): + tar_path.unlink() + registry.remove(name) + if not dry_run: + registry.save() + print_success(f"Removed {len(to_remove)} image(s) from registry and disk.") + else: + print_info(f"Dry-run: would remove {len(to_remove)} image(s).") + return if not clean_all: - print_info("Nothing to clean. Use --all to remove localci images.") + print_info("Nothing to clean. Use --all, --older-than, or --unused.") return + # --all: remove all capy Docker images (and optionally registry entries) result = _run(["docker", "image", "ls", "--format", "{{.Repository}}:{{.Tag}}"]) if result.returncode != 0: print_error(result.stderr.strip() or "Failed to list Docker images.") diff --git a/cli/localci/cli/run.py b/cli/localci/cli/run.py index 1c55317..115169d 100644 --- a/cli/localci/cli/run.py +++ b/cli/localci/cli/run.py @@ -284,6 +284,8 @@ def run( default_secrets={"GITHUB_TOKEN": gh_token}, default_env={}, ) + registry_path = project_dir / "image-registry.yml" + images_dir = project_dir / "images" / "capy" orchestrator = ParallelExecutionManager( queue=queue, workflow_file=workflow_path, @@ -294,6 +296,8 @@ def run( cache_config=cfg.cache, no_cache=no_cache, cache_dir_override=cache_dir, + registry_path=registry_path, + images_dir=images_dir, ) status_file = logs_dir / "last-status.json" diff --git a/cli/localci/core/__init__.py b/cli/localci/core/__init__.py index d37d00b..ceed8a6 100644 --- a/cli/localci/core/__init__.py +++ b/cli/localci/core/__init__.py @@ -38,6 +38,10 @@ extra_marks, select_image, ) +from localci.core.image_manager import ( # noqa: F401 + ImageManager, + image_name_from_entry, +) from localci.core.orchestrator import ( # noqa: F401 ExecutionRun, OrchestratorConfig, diff --git a/cli/localci/core/image_manager.py b/cli/localci/core/image_manager.py new file mode 100644 index 0000000..74a4e69 --- /dev/null +++ b/cli/localci/core/image_manager.py @@ -0,0 +1,365 @@ +"""Docker image management: load from .tar, create new images, save for reuse. + +Implements Issue 4: Docker Image Management per Design Guide. +- Load images from .tar files +- Create new images when no match found (needs_build) +- Save newly built images and update registry +- Image naming convention: -- +- Integrates with ImageRegistry (Issue 3) and DockerManager. +""" + +from __future__ import annotations + +import logging +import re +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from localci.core.registry import ( + ImageRegistry, + RegistryEntry, +) +from localci.utils.docker import DockerManager + +if TYPE_CHECKING: + from localci.core.models import QueuedJob + from localci.core.workflow import MatrixEntry + +logger = logging.getLogger(__name__) + +# Default project name for image naming (e.g. capy) +DEFAULT_PROJECT = "capy" + + +def image_name_from_entry(entry: "MatrixEntry", project: str = DEFAULT_PROJECT) -> str: + """Derive image name from matrix entry (no :latest). + + Naming convention: -- per Design Guide E.4. + Examples: capy-ubuntu-25.04-gcc15, capy-ubuntu-24.04-clang20-asan. + """ + if entry.container.image: + img = entry.container.image.strip().lower() + os_label = img.replace(":", "-", 1) if ":" in img else img + else: + os_label = (entry.runs_on or "ubuntu-latest").strip().lower() + compiler_label = f"{entry.compiler.family.value}{entry.compiler.version}" + base = f"{project}-{os_label}-{compiler_label}" + if entry.variant.coverage: + base += "-cov" + elif entry.variant.asan: + base += "-asan" + elif entry.variant.x86: + base += "-x86" + return base + + +def image_name_base_from_entry(entry: "MatrixEntry", project: str = DEFAULT_PROJECT) -> str: + """Derive base-only image name (OS + compiler, no variant suffix). + + Use when building a single base image per OS+toolchain for all variants + (standard, asan, x86, cov); variants use the same image with different flags. + """ + if entry.container.image: + img = entry.container.image.strip().lower() + os_label = img.replace(":", "-", 1) if ":" in img else img + else: + os_label = (entry.runs_on or "ubuntu-latest").strip().lower() + compiler_label = f"{entry.compiler.family.value}{entry.compiler.version}" + return f"{project}-{os_label}-{compiler_label}" + + +def _sanitize_name_for_dockerfile(name: str) -> str: + """Dockerfile filenames: allow alphanumeric, dash, dot.""" + return re.sub(r"[^a-zA-Z0-9.-]", "-", name) + + +class ImageManager: + """Prepare Docker images for jobs: load from .tar, build when no match, save for reuse.""" + + def __init__( + self, + project_dir: Path, + registry_path: Path, + images_dir: Optional[Path] = None, + project: str = DEFAULT_PROJECT, + docker: Optional[DockerManager] = None, + ) -> None: + self.project_dir = Path(project_dir).resolve() + self.registry_path = Path(registry_path) + self.images_dir = Path(images_dir) if images_dir else (self.project_dir / "images" / project) + self.project = project + self._docker = docker or DockerManager() + + def _registry(self) -> ImageRegistry: + reg = ImageRegistry(self.registry_path) + reg.load() + return reg + + def _resolve_tar_path(self, file_path: str) -> Path: + """Resolve registry 'file' (relative or absolute) to Path.""" + p = Path(file_path) + if not p.is_absolute(): + p = self.project_dir / p + return p.resolve() + + def _load_from_tar(self, entry: RegistryEntry, target_tag: Optional[str] = None) -> bool: + """Load image from registry entry's .tar file; optionally tag as target_tag.""" + tar_path = self._resolve_tar_path(entry.file) + ok, output = self._docker.load_image(tar_path) + if not ok: + logger.error("Load failed: %s", output) + return False + ref = DockerManager.parse_load_output(output) + if ref and target_tag and ref != target_tag: + if ref.startswith("sha256:"): + self._docker.tag_image(ref, target_tag) + elif ref != target_tag: + self._docker.tag_image(ref, target_tag) + return True + + def prepare_image_for_job(self, job: "QueuedJob") -> Optional[str]: + """Ensure the image for this job is available; load or build as needed. + + Returns the image tag to use for act (e.g. capy-ubuntu-25.04-gcc15:latest), + or None on failure. + """ + registry = self._registry() + if job.needs_build: + base_entry = None + if job.base_image_tag: + for e in registry.entries: + if e.docker_tag == job.base_image_tag: + base_entry = e + break + return self._build_new_image(job, registry, base_entry) + + # Use existing image: find registry entry by docker_tag and load from .tar if needed + entry = None + for e in registry.entries: + if e.docker_tag == job.image_tag: + entry = e + break + if not entry: + logger.warning("Image tag %s not in registry; assuming pre-loaded", job.image_tag) + if self._docker.image_exists(job.image_tag or ""): + return job.image_tag + return None + if self._docker.image_exists(job.image_tag or ""): + registry.update_usage(entry.name) + try: + registry.save() + except Exception as e: + logger.debug("Could not save registry usage: %s", e) + return job.image_tag + if not self._load_from_tar(entry, job.image_tag): + return None + registry.update_usage(entry.name) + try: + registry.save() + except Exception as e: + logger.debug("Could not save registry: %s", e) + return job.image_tag + + def _build_new_image( + self, + job: "QueuedJob", + registry: ImageRegistry, + base_entry: Optional[RegistryEntry], + ) -> Optional[str]: + """Build a new image when no full match exists; save to .tar and add to registry. + + Uses base-only naming (OS + compiler, no variant) so one image serves + all variants (standard, asan, x86, cov); variants use workflow flags. + """ + entry = job.matrix_entry + image_name = image_name_base_from_entry(entry, self.project) + image_tag = f"{image_name}:latest" + + if self._docker.image_exists(image_tag): + logger.debug("Base image already exists: %s", image_tag) + return image_tag + + # Ensure base image is loaded + if base_entry: + if not self._docker.image_exists(base_entry.docker_tag): + if not self._load_from_tar(base_entry): + logger.error("Failed to load base image %s", base_entry.docker_tag) + return None + base_from = base_entry.docker_tag + else: + base_from = None # Will use official base from entry (e.g. ubuntu:25.04) + + dockerfile_path = self._find_dockerfile(image_name) + if dockerfile_path and dockerfile_path.exists(): + ok = self._build_with_dockerfile(dockerfile_path, image_tag) + else: + ok = self._build_from_generated(entry, image_tag, base_from) + + if not ok: + return None + + # Save to .tar and add to registry + tar_rel = f"images/{self.project}/{image_name}.tar" + tar_path = self.project_dir / tar_rel + tar_path.parent.mkdir(parents=True, exist_ok=True) + ok_save, err = self._docker.save_image(image_tag, tar_path) + if not ok_save: + logger.error("Failed to save image: %s", err) + return image_tag # Image exists in Docker; still usable + + try: + file_rel = str(tar_path.relative_to(self.project_dir)) + except ValueError: + file_rel = str(tar_path) + new_entry = self._registry_entry_for_built(entry, image_name, file_rel) + existing = registry.find_by_name(new_entry.name) + if existing: + registry.remove(new_entry.name) + registry.add(new_entry) + registry.save() + return image_tag + + def _find_dockerfile(self, image_name: str) -> Optional[Path]: + """Look for Dockerfile. or Dockerfile. in images_dir.""" + safe = _sanitize_name_for_dockerfile(image_name) + candidates = [ + self.images_dir / f"Dockerfile.{image_name}", + self.images_dir / f"Dockerfile.{safe}", + ] + for p in candidates: + if p.exists(): + return p + return None + + def _build_with_dockerfile(self, dockerfile_path: Path, image_tag: str) -> bool: + """Run docker build -f -t .""" + import subprocess + context = dockerfile_path.parent + cmd = [ + "docker", "build", + "-f", str(dockerfile_path), + "-t", image_tag, + str(context), + ] + logger.info("Building image with Dockerfile: %s", dockerfile_path.name) + result = subprocess.run(cmd, capture_output=True, text=True, timeout=1800) + if result.returncode != 0: + logger.error("Build failed: %s", result.stderr) + return False + return True + + def _build_from_generated( + self, + entry: "MatrixEntry", + image_tag: str, + base_from: Optional[str], + ) -> bool: + """Generate a minimal Dockerfile and build (FROM base, install compiler + packages).""" + if not base_from: + # Use container image from matrix as base + base_from = (entry.container.image or "ubuntu:24.04").strip() + req_os = entry.container.image or "ubuntu:24.04" + compiler_family = entry.compiler.family.value + compiler_ver = entry.compiler.version or "13" + apt_packages = list(getattr(entry.packages, "apt_packages", None) or []) + build_tools = list(getattr(entry.packages, "build_tools", None) or []) + all_pkgs = list(dict.fromkeys(apt_packages + build_tools)) + # Add compiler packages + if compiler_family == "gcc": + all_pkgs.append(f"gcc-{compiler_ver}") + all_pkgs.append(f"g++-{compiler_ver}") + elif compiler_family == "clang": + all_pkgs.append(f"clang-{compiler_ver}") + all_pkgs.append(f"clang-tools-{compiler_ver}") + all_pkgs = [p for p in all_pkgs if p] + run_apt = ( + "RUN apt-get update && apt-get install -y --no-install-recommends " + + " ".join(all_pkgs) + + " && rm -rf /var/lib/apt/lists/*" + ) if all_pkgs else "" + set_env = "" + if compiler_family == "gcc": + set_env = f'ENV CC=gcc-{compiler_ver} CXX=g++-{compiler_ver}' + elif compiler_family == "clang": + set_env = f'ENV CC=clang-{compiler_ver} CXX=clang++-{compiler_ver}' + lines = [ + f"FROM {base_from}", + "RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*", + ] + if run_apt: + lines.append(run_apt) + if set_env: + lines.append(set_env) + content = "\n".join(lines) + with tempfile.NamedTemporaryFile(mode="w", suffix=".Dockerfile", delete=False) as f: + f.write(content) + df_path = Path(f.name) + try: + import subprocess + # Use images_dir as context (no COPY in generated Dockerfile) + context = str(self.images_dir) if self.images_dir.exists() else str(df_path.parent) + cmd = ["docker", "build", "-f", str(df_path), "-t", image_tag, context] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=1800) + if result.returncode != 0: + logger.error("Generated build failed: %s", result.stderr) + return False + return True + finally: + df_path.unlink(missing_ok=True) + + def _registry_entry_for_built( + self, + entry: "MatrixEntry", + image_name: str, + file_rel: str, + ) -> RegistryEntry: + """Build a RegistryEntry for a newly built image.""" + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + size_mb = None + if self._docker.image_exists(f"{image_name}:latest"): + size_mb = int(self._docker.image_size(f"{image_name}:latest") or 0) + req_os = (entry.container.image or "ubuntu:24.04").strip().lower() + arch = getattr(entry, "architecture", "x86_64") or "x86_64" + compilers = [f"{entry.compiler.family.value}-{entry.compiler.version}"] + apt_packages = list(getattr(entry.packages, "apt_packages", None) or []) + tools = list(getattr(entry.packages, "build_tools", None) or []) + return RegistryEntry( + name=image_name, + file=file_rel, + docker_tag=f"{image_name}:latest", + os=req_os, + architecture=arch, + packages=apt_packages, + compilers=compilers, + tools=tools, + size_mb=size_mb, + created=now, + last_used=now, + usage_count=0, + ) + + def save_image_to_tar( + self, + image_tag: str, + tar_path: Path, + registry: Optional[ImageRegistry] = None, + entry: Optional[RegistryEntry] = None, + ) -> bool: + """Save a Docker image to .tar; optionally add/update registry.""" + ok, err = self._docker.save_image(image_tag, Path(tar_path)) + if not ok: + logger.error("Save failed: %s", err) + return False + if registry and entry: + try: + existing = registry.find_by_name(entry.name) + if existing: + registry.update(entry.name, file=str(tar_path), last_used=entry.last_used) + else: + registry.add(entry) + registry.save() + except Exception as e: + logger.warning("Could not update registry: %s", e) + return True diff --git a/cli/localci/core/orchestrator.py b/cli/localci/core/orchestrator.py index e969afe..2f131c7 100644 --- a/cli/localci/core/orchestrator.py +++ b/cli/localci/core/orchestrator.py @@ -17,6 +17,7 @@ from localci.core.command_builder import ActCommandBuilder from localci.core.config import resolve_cache_paths from localci.core.cmake_cache import compute_cmake_input_digest +from localci.core.image_manager import ImageManager from localci.core.workflow import MatrixEntry from localci.core.executor import JobExecutor, JobResult, JobStatus from localci.core.models import JobEvent, JobEventType, QueuedJob @@ -139,6 +140,8 @@ def __init__( cache_config: Optional["CacheConfig"] = None, no_cache: bool = False, cache_dir_override: Optional[Path] = None, + registry_path: Optional[Path] = None, + images_dir: Optional[Path] = None, ): self.queue = queue self.workflow_file = Path(workflow_file) @@ -154,6 +157,8 @@ def __init__( if cache_dir_override is not None else None ) + self._registry_path = Path(registry_path) if registry_path else (self.project_dir / "image-registry.yml") + self._images_dir = Path(images_dir) if images_dir else (self.project_dir / "images" / "capy") self._executor = JobExecutor(logs_dir=self.logs_dir) self._docker = DockerManager() @@ -416,6 +421,22 @@ def _prepare_image(self, job: QueuedJob) -> Optional[str]: if self._docker.image_exists(job.image_tag): logger.debug("Image already loaded: %s", job.image_tag) return job.image_tag + # Use ImageManager when registry exists: load from .tar or build new image + if self._registry_path.exists(): + try: + img_mgr = ImageManager( + project_dir=self.project_dir, + registry_path=self._registry_path, + images_dir=self._images_dir, + docker=self._docker, + ) + tag = img_mgr.prepare_image_for_job(job) + if tag: + logger.info("Image ready: %s", tag) + return tag + except Exception as e: + logger.exception("Image preparation failed: %s", e) + raise logger.info("Image ready: %s", job.image_tag) return job.image_tag diff --git a/cli/localci/core/progress.py b/cli/localci/core/progress.py index 3197911..7d81cbc 100644 --- a/cli/localci/core/progress.py +++ b/cli/localci/core/progress.py @@ -9,6 +9,7 @@ import json import logging import threading +import time from dataclasses import dataclass, field from datetime import datetime from pathlib import Path diff --git a/cli/localci/core/queue_builder.py b/cli/localci/core/queue_builder.py index d0aa0fb..a9f344e 100644 --- a/cli/localci/core/queue_builder.py +++ b/cli/localci/core/queue_builder.py @@ -24,7 +24,7 @@ def _resolve_image_tag_and_build( ) -> tuple[str, Optional[str], bool]: """Resolve (image_tag, base_image_tag, needs_build) via registry matching, or derive tag and no build.""" if not registry_path or not registry_path.exists(): - return _derive_image_tag(entry), None, False + return _derive_image_tag_base(entry), None, False from localci.core.registry import ImageRegistry registry = ImageRegistry(registry_path) @@ -33,8 +33,8 @@ def _resolve_image_tag_and_build( if result.use_image: return result.use_image.docker_tag, None, False if result.base_image: - return _derive_image_tag(entry), result.base_image.docker_tag, True - return _derive_image_tag(entry), None, True + return _derive_image_tag_base(entry), result.base_image.docker_tag, True + return _derive_image_tag_base(entry), None, True def _derive_image_tag(entry: MatrixEntry) -> str: @@ -55,6 +55,21 @@ def _derive_image_tag(entry: MatrixEntry) -> str: return f"{base}:latest" +def _derive_image_tag_base(entry: MatrixEntry) -> str: + """Derive base-only Docker image tag (OS + compiler, no variant). + + Used when needs_build so one image per OS+toolchain is built and reused + for all variants (standard, asan, x86, cov). + """ + if entry.container.image: + img = entry.container.image.strip().lower() + os_label = img.replace(":", "-", 1) if ":" in img else img + else: + os_label = entry.runs_on + compiler_label = f"{entry.compiler.family.value}{entry.compiler.version}" + return f"capy-{os_label}-{compiler_label}:latest" + + def _matches_filter(entry: MatrixEntry, filters: list[dict]) -> bool: """True if entry matches any of the filter dicts.""" for f in filters: diff --git a/cli/localci/utils/docker.py b/cli/localci/utils/docker.py index 001b05b..504f189 100644 --- a/cli/localci/utils/docker.py +++ b/cli/localci/utils/docker.py @@ -70,7 +70,9 @@ def _docker_cmd(self, *args: str) -> list[str]: def load_image(self, tar_path: Path) -> tuple[bool, str]: """Load Docker image from a ``.tar`` file. - Returns ``(success, image_id_or_error)``. + Returns ``(success, output_or_error)``. On success, output is the raw + stdout (e.g. "Loaded image: repo:tag" or "Loaded image ID: sha256:..."). + Use parse_load_output() to get an image reference for tagging. """ if not tar_path.exists(): return False, f"Image file not found: {tar_path}" @@ -94,6 +96,45 @@ def load_image(self, tar_path: Path) -> tuple[bool, str]: logger.info("Loaded in %.1fs: %s", duration, output) return True, output + @staticmethod + def parse_load_output(load_stdout: str) -> Optional[str]: + """Parse docker load stdout to get image reference for tagging. + + - "Loaded image: repo:tag" -> "repo:tag" + - "Loaded image ID: sha256:abc..." -> "sha256:abc..." + - Multiple lines (e.g. multiple images in tar) -> first image ref + Returns None if no recognized pattern. + """ + for line in load_stdout.strip().splitlines(): + line = line.strip() + if line.startswith("Loaded image: "): + return line.replace("Loaded image: ", "", 1).strip() + if line.startswith("Loaded image ID: "): + return line.replace("Loaded image ID: ", "", 1).strip() + return None + + def save_image(self, image_ref: str, tar_path: Path) -> tuple[bool, str]: + """Save a Docker image to a ``.tar`` file. + + image_ref: image name:tag or ID (e.g. capy-ubuntu-25.04-gcc15:latest). + Returns (success, error_message). On success error_message is empty. + """ + tar_path = Path(tar_path) + tar_path.parent.mkdir(parents=True, exist_ok=True) + logger.info("Saving image %s to %s", image_ref, tar_path) + start = time.time() + result = subprocess.run( + self._docker_cmd("save", "-o", str(tar_path), image_ref), + capture_output=True, + text=True, + timeout=600, # 10 min for large images + ) + duration = time.time() - start + if result.returncode != 0: + return False, result.stderr.strip() or "docker save failed" + logger.info("Saved in %.1fs", duration) + return True, "" + def tag_image(self, source: str, target: str) -> bool: """Tag a Docker image.""" result = subprocess.run( diff --git a/cli/tests/test_executor.py b/cli/tests/test_executor.py index 0087c01..48131c0 100644 --- a/cli/tests/test_executor.py +++ b/cli/tests/test_executor.py @@ -1018,6 +1018,39 @@ def test_has_docker_property(self, mock_which, mock_run): dm = DockerManager() assert dm.has_docker is True + def test_parse_load_output_image_tag(self): + from localci.utils.docker import DockerManager + + out = "Loaded image: capy-ubuntu-25.04-gcc15:latest" + assert DockerManager.parse_load_output(out) == "capy-ubuntu-25.04-gcc15:latest" + + def test_parse_load_output_image_id(self): + from localci.utils.docker import DockerManager + + out = "Loaded image ID: sha256:abc123def" + assert DockerManager.parse_load_output(out) == "sha256:abc123def" + + def test_parse_load_output_empty(self): + from localci.utils.docker import DockerManager + + assert DockerManager.parse_load_output("") is None + assert DockerManager.parse_load_output("Some other line") is None + + @patch("subprocess.run") + @patch("shutil.which") + def test_save_image_success(self, mock_which, mock_run, tmp_path): + mock_which.return_value = "/usr/bin/docker" + mock_run.return_value = MagicMock(returncode=0, stdout="docker 24.0") + + from localci.utils.docker import DockerManager + + dm = DockerManager() + out_path = tmp_path / "out.tar" + mock_run.return_value = MagicMock(returncode=0) + ok, err = dm.save_image("capy-ubuntu-25.04-gcc15:latest", out_path) + assert ok is True + assert err == "" + # ===================================================================== # ExecutionSummary tests diff --git a/cli/tests/test_image_manager.py b/cli/tests/test_image_manager.py new file mode 100644 index 0000000..2784527 --- /dev/null +++ b/cli/tests/test_image_manager.py @@ -0,0 +1,179 @@ +"""Tests for Docker image management (Issue 4).""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from localci.core.image_manager import ( + ImageManager, + image_name_base_from_entry, + image_name_from_entry, +) +from localci.core.workflow import ( + BuildVariant, + CompilerFamily, + CompilerInfo, + ContainerInfo, + MatrixEntry, + PackageRequirements, + Platform, + BuildSystem, +) + + +def _make_entry( + container_image: str = "ubuntu:25.04", + compiler_family: str = "gcc", + compiler_version: str = "15", + coverage: bool = False, + asan: bool = False, + x86: bool = False, +) -> MatrixEntry: + family = CompilerFamily.GCC if compiler_family == "gcc" else CompilerFamily.CLANG + return MatrixEntry( + index=0, + name="GCC 15", + platform=Platform.LINUX, + compiler=CompilerInfo( + family=family, + version=compiler_version, + ), + container=ContainerInfo(image=container_image), + variant=BuildVariant(coverage=coverage, asan=asan, x86=x86), + packages=PackageRequirements(), + runs_on="ubuntu-latest", + build_system=BuildSystem.CMAKE, + architecture="x86_64", + ) + + +class TestImageNameFromEntry: + """image_name_from_entry naming convention.""" + + def test_ubuntu_gcc(self): + entry = _make_entry(container_image="ubuntu:25.04", compiler_family="gcc", compiler_version="15") + assert image_name_from_entry(entry) == "capy-ubuntu-25.04-gcc15" + assert image_name_from_entry(entry, project="beast2") == "beast2-ubuntu-25.04-gcc15" + + def test_ubuntu_clang(self): + entry = _make_entry(container_image="ubuntu:24.04", compiler_family="clang", compiler_version="20") + assert image_name_from_entry(entry) == "capy-ubuntu-24.04-clang20" + + def test_variant_coverage(self): + entry = _make_entry(coverage=True) + assert image_name_from_entry(entry) == "capy-ubuntu-25.04-gcc15-cov" + + def test_variant_asan(self): + entry = _make_entry(asan=True) + assert image_name_from_entry(entry) == "capy-ubuntu-25.04-gcc15-asan" + + def test_variant_x86(self): + entry = _make_entry(x86=True) + assert image_name_from_entry(entry) == "capy-ubuntu-25.04-gcc15-x86" + + +class TestImageNameBaseFromEntry: + """image_name_base_from_entry: base-only (OS + compiler), no variant suffix.""" + + def test_base_no_variant(self): + entry = _make_entry(container_image="ubuntu:25.04", compiler_family="gcc", compiler_version="15") + assert image_name_base_from_entry(entry) == "capy-ubuntu-25.04-gcc15" + + def test_base_ignores_asan(self): + entry = _make_entry(asan=True) + assert image_name_base_from_entry(entry) == "capy-ubuntu-25.04-gcc15" + + def test_base_ignores_cov_and_x86(self): + entry = _make_entry(coverage=True) + assert image_name_base_from_entry(entry) == "capy-ubuntu-25.04-gcc15" + entry_x86 = _make_entry(x86=True) + assert image_name_base_from_entry(entry_x86) == "capy-ubuntu-25.04-gcc15" + + +class TestImageManager: + """ImageManager with mocked Docker.""" + + @patch("localci.core.image_manager.DockerManager") + def test_prepare_image_use_existing_loaded(self, mock_docker_cls, tmp_path): + registry_path = tmp_path / "image-registry.yml" + registry_path.write_text(""" +version: "1.0" +images: + - name: capy-ubuntu-25.04-gcc15 + file: images/capy/capy-ubuntu-25.04-gcc15.tar + docker_tag: capy-ubuntu-25.04-gcc15:latest + os: ubuntu:25.04 + architecture: x86_64 + compilers: [gcc-15] +""") + mock_docker = MagicMock() + mock_docker.image_exists.return_value = True + mock_docker_cls.return_value = mock_docker + + from localci.core.models import QueuedJob + + entry = _make_entry() + job = QueuedJob( + job_id="build", + matrix_entry=entry, + priority=1, + dependencies=[], + image_tag="capy-ubuntu-25.04-gcc15:latest", + base_image_tag=None, + needs_build=False, + ) + mgr = ImageManager( + project_dir=tmp_path, + registry_path=registry_path, + docker=mock_docker, + ) + tag = mgr.prepare_image_for_job(job) + assert tag == "capy-ubuntu-25.04-gcc15:latest" + mock_docker.load_image.assert_not_called() + + @patch("localci.core.image_manager.DockerManager") + def test_prepare_image_load_from_tar(self, mock_docker_cls, tmp_path): + registry_path = tmp_path / "image-registry.yml" + registry_path.write_text(""" +version: "1.0" +images: + - name: capy-ubuntu-25.04-gcc15 + file: images/capy/capy-ubuntu-25.04-gcc15.tar + docker_tag: capy-ubuntu-25.04-gcc15:latest + os: ubuntu:25.04 + architecture: x86_64 +""") + tar_path = tmp_path / "images" / "capy" / "capy-ubuntu-25.04-gcc15.tar" + tar_path.parent.mkdir(parents=True, exist_ok=True) + tar_path.write_bytes(b"fake") + + mock_docker = MagicMock() + mock_docker.image_exists.return_value = False + mock_docker.load_image.return_value = (True, "Loaded image: capy-ubuntu-25.04-gcc15:latest") + mock_docker_cls.return_value = mock_docker + + from localci.core.models import QueuedJob + + entry = _make_entry() + job = QueuedJob( + job_id="build", + matrix_entry=entry, + priority=1, + dependencies=[], + image_tag="capy-ubuntu-25.04-gcc15:latest", + base_image_tag=None, + needs_build=False, + ) + mgr = ImageManager( + project_dir=tmp_path, + registry_path=registry_path, + docker=mock_docker, + ) + tag = mgr.prepare_image_for_job(job) + assert tag == "capy-ubuntu-25.04-gcc15:latest" + mock_docker.load_image.assert_called_once() + call_path = mock_docker.load_image.call_args[0][0] + assert str(call_path).endswith("capy-ubuntu-25.04-gcc15.tar")