diff --git a/.github/workflows/frameos-cross-toolchain.yml b/.github/workflows/frameos-cross-toolchain.yml index 60d539676..61dffbe5b 100644 --- a/.github/workflows/frameos-cross-toolchain.yml +++ b/.github/workflows/frameos-cross-toolchain.yml @@ -56,8 +56,15 @@ jobs: import os import re import subprocess + import sys from pathlib import Path + sys.path.insert(0, str(Path("backend").resolve())) + from app.utils.cross_toolchain_packages import ( # noqa: E402 + TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS, + TARGET_CROSS_TOOLCHAIN_PACKAGES, + ) + def sanitize(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", value) @@ -72,6 +79,11 @@ jobs: safe_platform = sanitize(target["platform"].replace("/", "_")) image = f"{image_repo}:{safe_base}-{safe_platform}-{image_tag}" metadata_file = Path(f"/tmp/toolchain-metadata-{index}.json") + target_cross_dpkg_archs = "" + target_cross_packages = "" + if target["platform"] == "linux/amd64": + target_cross_dpkg_archs = " ".join(TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS) + target_cross_packages = " ".join(TARGET_CROSS_TOOLCHAIN_PACKAGES) subprocess.run( [ @@ -82,6 +94,10 @@ jobs: target["platform"], "--build-arg", f"BASE_IMAGE={base_image}", + "--build-arg", + f"TARGET_CROSS_DPKG_ARCHS={target_cross_dpkg_archs}", + "--build-arg", + f"TARGET_CROSS_PACKAGES={target_cross_packages}", "--tag", image, "--push", diff --git a/backend/app/api/frames.py b/backend/app/api/frames.py index fea570cf5..8dc05838f 100644 --- a/backend/app/api/frames.py +++ b/backend/app/api/frames.py @@ -112,6 +112,9 @@ ) from app.models.assets import copy_custom_fonts_to_local_source_folder from app.models.settings import get_settings_dict +from app.utils.build_environment import selected_build_environment_provider +from app.utils.build_host import get_build_executor_config +from app.utils.build_executor import build_environment_requires_executor_config from app.utils.ssh_key_utils import default_ssh_key_ids from app.utils.timezone import frame_timezone, normalize_timezone, stored_timezone from app.utils.tls import generate_frame_tls_material, parse_certificate_not_valid_after @@ -2436,6 +2439,15 @@ async def api_frame_buildroot_sd_image( if (frame.mode or "rpios") != "buildroot": _bad_request("SD card image generation is only available for Buildroot frames") + settings = get_settings_dict(db, project_id=frame.project_id) + build_environment_provider = selected_build_environment_provider(settings) + if build_environment_provider == "none": + _bad_request( + "Buildroot SD card image generation requires Docker, build host, or Modal sandboxes as the global build environment." + ) + if build_environment_requires_executor_config(build_environment_provider) and get_build_executor_config(db, frame.project_id) is None: + _bad_request(f"Selected build environment '{build_environment_provider}' is not configured") + try: ensure_buildroot_frame_defaults(frame) except ValueError as exc: diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 463765d24..6466ccab8 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -1,4 +1,6 @@ from http import HTTPStatus +from types import SimpleNamespace + from fastapi import Depends, HTTPException from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.orm import Session @@ -7,6 +9,10 @@ from app.models.settings import get_settings_dict, Settings from app.schemas.settings import SettingsResponse, SettingsUpdateRequest from app.tenancy import current_project_id +from app.utils.build_environment import selected_build_environment_provider +from app.utils.build_executor import create_build_executor +from app.utils.build_host import BuildHostConfig +from app.utils.modal_sandbox import ModalSandboxConfig from app.utils.posthog import initialize_posthog from . import api_project @@ -23,6 +29,13 @@ async def set_settings(data: SettingsUpdateRequest, db: Session = Depends(get_db try: current_settings = get_settings_dict(db, project_id=project_id) + merged_settings = {**current_settings, **payload} + provider = selected_build_environment_provider(merged_settings) + if isinstance(payload.get("buildHost"), dict): + payload["buildHost"] = {**payload["buildHost"], "enabled": provider == "buildHost"} + if isinstance(payload.get("modalSandbox"), dict): + payload["modalSandbox"] = {**payload["modalSandbox"], "enabled": provider == "modal"} + for key, value in payload.items(): if value != current_settings.get(key): setting = db.query(Settings).filter_by(project_id=project_id, key=key).first() @@ -39,3 +52,76 @@ async def set_settings(data: SettingsUpdateRequest, db: Session = Depends(get_db if "posthog" in payload: initialize_posthog(updated_settings, project_id=project_id) return updated_settings + + +@api_project.post("/settings/test_build_host") +async def test_build_host(data: SettingsUpdateRequest): + payload = data.to_dict() + raw_build_host_settings = payload.get("buildHost") if isinstance(payload, dict) else None + build_host_config = BuildHostConfig.from_settings( + {**raw_build_host_settings, "enabled": True} if isinstance(raw_build_host_settings, dict) else raw_build_host_settings + ) + if build_host_config is None: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Select build host via SSH and enter a host, user, and private SSH key first", + ) + + try: + async with create_build_executor( + build_host_config, + db=None, + redis=None, + frame=SimpleNamespace(id=0), + workspace_prefix="frameos-build-host-test-", + ) as executor: + status, out, err = await executor.run( + "echo frameos-build-host-ok && command -v docker >/dev/null && docker buildx version >/dev/null", + log_command=False, + log_output=False, + ) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=HTTPStatus.BAD_GATEWAY, detail=f"Build host connection failed: {exc}") from exc + + if status != 0: + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail=err or out or "Build host is missing Docker or the Docker Buildx plugin", + ) + + return {"ok": True, "output": (out or "").strip()} + + +@api_project.post("/settings/test_modal_sandbox") +async def test_modal_sandbox(data: SettingsUpdateRequest): + payload = data.to_dict() + raw_modal_settings = payload.get("modalSandbox") if isinstance(payload, dict) else None + modal_config = ModalSandboxConfig.from_settings(raw_modal_settings) + if modal_config is None: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Select Modal sandboxes and enter a token ID and token secret first", + ) + + try: + async with create_build_executor( + modal_config, + db=None, + redis=None, + frame=SimpleNamespace(id=0), + ) as executor: + status, out, err = await executor.run( + "command -v nimble && nimble --version >/dev/null && echo frameos-modal-sandbox-ok", + log_command=False, + log_output=False, + ) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=HTTPStatus.BAD_GATEWAY, detail=f"Modal sandbox test failed: {exc}") from exc + + if status != 0: + raise HTTPException( + status_code=HTTPStatus.BAD_GATEWAY, + detail=err or out or "Modal sandbox image is missing the FrameOS Nim toolchain", + ) + + return {"ok": True, "output": (out or "").strip()} diff --git a/backend/app/api/system.py b/backend/app/api/system.py index 1b5807b65..2ec79c1c2 100644 --- a/backend/app/api/system.py +++ b/backend/app/api/system.py @@ -1,5 +1,5 @@ from app.api import api_user -from app.schemas.system import CacheInfo, DatabaseInfo, DiskInfo, LoadInfo, MemoryInfo, SystemInfoResponse, SystemMetricsResponse +from app.schemas.system import CacheInfo, DatabaseInfo, DiskInfo, DockerInfo, LoadInfo, MemoryInfo, SystemInfoResponse, SystemMetricsResponse from app.utils.system_info import get_system_info, get_system_metrics @@ -27,15 +27,24 @@ def _database_to_schema(database) -> DatabaseInfo: ) +def _docker_to_schema(docker) -> DockerInfo: + return DockerInfo( + cliAvailable=docker.cli_available, + daemonAvailable=docker.daemon_available, + error=docker.error, + ) + + @api_user.get("/system/info", response_model=SystemInfoResponse) def system_info(): - disk, caches, database, memory, load = get_system_info() + disk, caches, database, memory, load, docker = get_system_info() return SystemInfoResponse( disk=_disk_to_schema(disk), caches=_cache_to_schema(caches), database=_database_to_schema(database), memory=_memory_to_schema(memory), load=_load_to_schema(load), + docker=_docker_to_schema(docker), ) diff --git a/backend/app/api/tests/test_frames.py b/backend/app/api/tests/test_frames.py index b65a6fd66..0e526fefc 100644 --- a/backend/app/api/tests/test_frames.py +++ b/backend/app/api/tests/test_frames.py @@ -17,6 +17,7 @@ from app.models.frame import Frame from app.models.log import Log from app.models.metrics import Metrics +from app.models.settings import Settings from app.models.user import User from app.tenancy import ensure_default_project_for_user from app.tasks.buildroot_image import BUILDROOT_SD_IMAGE_CUSTOMIZATION_VERSION, buildroot_sd_image_config_fingerprint @@ -1167,6 +1168,47 @@ async def fake_buildroot_sd_image(id, _redis, *, request_id=None, queue_job_id=N assert frame.agent['deployWithAgent'] is True +@pytest.mark.asyncio +async def test_api_frame_buildroot_sd_image_accepts_configured_build_host(async_client, db, redis, monkeypatch): + import app.tasks.buildroot_image as buildroot_image_module + + frame = await new_frame(db, redis, 'BuildrootFrame', 'frame.local', 'backend.local') + frame.mode = 'buildroot' + frame.network = { + **(frame.network or {}), + 'wifiSSID': 'Test WiFi', + 'wifiPassword': 'secret1234', + } + frame.buildroot = {'platform': 'raspberry-pi-zero-2-w'} + db.add(Settings(project_id=frame.project_id, key='buildEnvironment', value={'provider': 'buildHost'})) + db.add( + Settings( + project_id=frame.project_id, + key='buildHost', + value={ + 'enabled': True, + 'host': 'builder.local', + 'user': 'ubuntu', + 'sshKey': 'dummy-key', + }, + ) + ) + db.add(frame) + db.commit() + captured: list[int] = [] + + async def fake_buildroot_sd_image(id, _redis, *, request_id=None, queue_job_id=None): + captured.append(id) + + monkeypatch.setattr(buildroot_image_module, "buildroot_sd_image", fake_buildroot_sd_image) + + response = await async_client.post(f'/api/frames/{frame.id}/buildroot/sd_image') + + assert response.status_code == 200 + assert response.json()['message'] == 'Buildroot SD card image preparation started' + assert captured == [frame.id] + + @pytest.mark.asyncio async def test_api_frame_buildroot_sd_image_does_not_publish_previous_error(async_client, db, redis, monkeypatch): import app.api.frames as frames_api_module diff --git a/backend/app/api/tests/test_settings.py b/backend/app/api/tests/test_settings.py index c5e2f4617..325054ff3 100644 --- a/backend/app/api/tests/test_settings.py +++ b/backend/app/api/tests/test_settings.py @@ -17,8 +17,149 @@ async def test_set_settings(async_client): assert updated["some_setting"] == "hello" +@pytest.mark.asyncio +async def test_set_settings_normalizes_build_environment_enabled_flags(async_client): + payload = { + "buildEnvironment": {"provider": "modal"}, + "buildHost": {"enabled": True, "host": "builder.local"}, + "modalSandbox": {"tokenId": "ak-test", "tokenSecret": "as-test"}, + } + response = await async_client.post('/api/settings', json=payload) + assert response.status_code == 200, f"Got {response.status_code} and {response.json()}" + updated = response.json() + assert updated["buildHost"]["enabled"] is False + assert updated["modalSandbox"]["enabled"] is True + + +@pytest.mark.asyncio +async def test_set_settings_normalizes_partial_provider_settings_from_current_settings(async_client): + initial_response = await async_client.post( + '/api/settings', + json={ + "buildEnvironment": {"provider": "buildHost"}, + "buildHost": {"enabled": True, "host": "builder.local"}, + }, + ) + assert initial_response.status_code == 200, ( + f"Got {initial_response.status_code} and {initial_response.json()}" + ) + + response = await async_client.post('/api/settings', json={"buildHost": {"host": "builder-2.local"}}) + assert response.status_code == 200, f"Got {response.status_code} and {response.json()}" + updated = response.json() + assert updated["buildEnvironment"]["provider"] == "buildHost" + assert updated["buildHost"]["enabled"] is True + assert updated["buildHost"]["host"] == "builder-2.local" + + @pytest.mark.asyncio async def test_set_settings_no_payload(async_client): response = await async_client.post('/api/settings', json={}) assert response.status_code == 400 assert response.json()['detail'] == "No JSON payload received" + + +@pytest.mark.asyncio +async def test_modal_sandbox_test_requires_credentials(async_client): + response = await async_client.post('/api/settings/test_modal_sandbox', json={"modalSandbox": {"enabled": True}}) + + assert response.status_code == 400 + assert response.json()["detail"] == "Select Modal sandboxes and enter a token ID and token secret first" + + +@pytest.mark.asyncio +async def test_build_host_test_requires_credentials(async_client): + response = await async_client.post('/api/settings/test_build_host', json={"buildHost": {"host": "builder.local"}}) + + assert response.status_code == 400 + assert response.json()["detail"] == "Select build host via SSH and enter a host, user, and private SSH key first" + + +@pytest.mark.asyncio +async def test_build_host_test_runs_probe(async_client, monkeypatch): + class FakeBuildExecutor: + def __init__(self, config, **kwargs): + self.config = config + self.kwargs = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def run(self, command, **kwargs): + assert command == "echo frameos-build-host-ok && command -v docker >/dev/null && docker buildx version >/dev/null" + assert kwargs["log_command"] is False + assert kwargs["log_output"] is False + return 0, "frameos-build-host-ok\n", None + + def fake_create_build_executor(config, **kwargs): + assert config.host == "builder.local" + assert config.user == "ubuntu" + assert config.ssh_key == "dummy-key" + assert kwargs["db"] is None + assert kwargs["redis"] is None + assert kwargs["frame"].id == 0 + assert kwargs["workspace_prefix"] == "frameos-build-host-test-" + return FakeBuildExecutor(config, **kwargs) + + monkeypatch.setattr("app.api.settings.create_build_executor", fake_create_build_executor) + + response = await async_client.post( + '/api/settings/test_build_host', + json={ + "buildHost": { + "host": "builder.local", + "user": "ubuntu", + "sshKey": "dummy-key", + } + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "output": "frameos-build-host-ok"} + + +@pytest.mark.asyncio +async def test_modal_sandbox_test_runs_probe(async_client, monkeypatch): + class FakeBuildExecutor: + def __init__(self, config, **kwargs): + self.config = config + self.kwargs = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def run(self, command, **kwargs): + assert command == "command -v nimble && nimble --version >/dev/null && echo frameos-modal-sandbox-ok" + assert kwargs["log_command"] is False + assert kwargs["log_output"] is False + return 0, "frameos-modal-sandbox-ok\n", None + + def fake_create_build_executor(config, **kwargs): + assert config.token_id == "ak-test" + assert config.token_secret == "as-test" + assert kwargs["db"] is None + assert kwargs["redis"] is None + assert kwargs["frame"].id == 0 + return FakeBuildExecutor(config, **kwargs) + + monkeypatch.setattr("app.api.settings.create_build_executor", fake_create_build_executor) + + response = await async_client.post( + '/api/settings/test_modal_sandbox', + json={ + "modalSandbox": { + "enabled": True, + "tokenId": "ak-test", + "tokenSecret": "as-test", + } + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "output": "frameos-modal-sandbox-ok"} diff --git a/backend/app/models/settings.py b/backend/app/models/settings.py index cb60cd48d..b5c184944 100644 --- a/backend/app/models/settings.py +++ b/backend/app/models/settings.py @@ -31,6 +31,8 @@ def default_settings() -> dict: "wifiSSID": "", "wifiPassword": "", }, + "buildEnvironment": {}, + "modalSandbox": {}, } diff --git a/backend/app/schemas/system.py b/backend/app/schemas/system.py index e1815018d..5b07d9da3 100644 --- a/backend/app/schemas/system.py +++ b/backend/app/schemas/system.py @@ -31,12 +31,19 @@ class DatabaseInfo(BaseModel): exists: bool +class DockerInfo(BaseModel): + cliAvailable: bool + daemonAvailable: bool + error: str | None = None + + class SystemInfoResponse(BaseModel): disk: DiskInfo caches: list[CacheInfo] database: DatabaseInfo memory: MemoryInfo load: LoadInfo + docker: DockerInfo class SystemMetricsResponse(BaseModel): diff --git a/backend/app/tasks/binary_builder.py b/backend/app/tasks/binary_builder.py index 122cdbbb0..994d0c8a5 100644 --- a/backend/app/tasks/binary_builder.py +++ b/backend/app/tasks/binary_builder.py @@ -26,7 +26,14 @@ precompiled_frameos_release_url, ) from app.tasks.prebuilt_deps import PrebuiltEntry, fetch_prebuilt_manifest, resolve_prebuilt_target -from app.utils.build_host import get_build_host_config +from app.models.settings import get_settings_dict +from app.utils.build_environment import selected_build_environment_provider +from app.utils.build_host import get_build_executor_config +from app.utils.build_executor import ( + build_executor_display_name, + build_executor_kind_name, + ensure_build_executor_configured, +) from app.utils.cross_compile import ( TargetMetadata, build_binary_with_cross_toolchain, @@ -46,6 +53,13 @@ ) +def get_build_host_config(*args, **kwargs): # noqa: ANN002, ANN003 + """Compatibility shim for older tests monkeypatching this module symbol.""" + from app.utils.build_host import get_build_host_config as _get_build_host_config + + return _get_build_host_config(*args, **kwargs) + + def should_suggest_clearing_build_cache(error_message: str) -> bool: normalized = error_message.lower() return any(hint in normalized for hint in LINKER_ERROR_HINTS) @@ -64,6 +78,8 @@ class FrameBinaryPlan: prebuilt_entry: PrebuiltEntry | None prebuilt_target: str | None requested_compilation_mode: str | None = None + build_executor: str | None = None + allow_on_device_fallback: bool = True will_attempt_precompiled: bool = False precompiled_release_url: str | None = None precompiled_skip_reason: str | None = None @@ -82,6 +98,8 @@ def to_dict(self) -> dict[str, object]: "force_cross_compile": self.force_cross_compile, "cross_compile_supported": self.cross_compile_supported, "build_host_configured": self.build_host_configured, + "build_executor": self.build_executor, + "allow_on_device_fallback": self.allow_on_device_fallback, "will_attempt_cross_compile": self.will_attempt_cross_compile, "prebuilt_target": self.prebuilt_target, "has_prebuilt_entry": self.prebuilt_entry is not None, @@ -176,6 +194,7 @@ async def plan_build( *, allow_cross_compile: bool = True, force_cross_compile: bool = False, + allow_on_device_fallback: bool = True, target_override: TargetMetadata | None = None, compilation_mode: str | None = None, ) -> FrameBinaryPlan: @@ -217,10 +236,15 @@ async def plan_build( ) project_id = getattr(self.frame, "project_id", None) - build_host = get_build_host_config(self.db, project_id) + build_executor = get_build_executor_config(self.db, project_id) + settings = get_settings_dict(self.db, project_id=project_id) if self.db and project_id is not None else {} + build_environment_provider = selected_build_environment_provider(settings) cross_compile_supported = can_cross_compile_target(target.arch) will_attempt_cross_compile = ( - allow_cross_compile and cross_compile_supported and not will_attempt_precompiled + allow_cross_compile + and build_environment_provider != "none" + and cross_compile_supported + and not will_attempt_precompiled ) return FrameBinaryPlan( @@ -230,7 +254,9 @@ async def plan_build( allow_cross_compile=allow_cross_compile, force_cross_compile=force_cross_compile, cross_compile_supported=cross_compile_supported, - build_host_configured=build_host is not None, + build_host_configured=build_executor is not None, + build_executor=build_executor_display_name(build_executor) if build_executor else None, + allow_on_device_fallback=allow_on_device_fallback, will_attempt_cross_compile=will_attempt_cross_compile, prebuilt_entry=prebuilt_entry, prebuilt_target=prebuilt_target, @@ -247,7 +273,9 @@ async def build( precompiled_install_all_drivers: bool = False, ) -> FrameBinaryBuildResult: project_id = getattr(self.frame, "project_id", None) - build_host = get_build_host_config(self.db, project_id) + build_executor = get_build_executor_config(self.db, project_id) + settings = get_settings_dict(self.db, project_id=project_id) if self.db and project_id is not None else {} + build_environment_provider = selected_build_environment_provider(settings) build_dir = create_build_folder(self.temp_dir, self.deployer.build_id) if plan.will_attempt_precompiled: await self._log("stdout", f"{icon} Using precompiled FrameOS release") @@ -304,10 +332,11 @@ async def build( cross_compiled = False binary_path: str | None = None if plan.will_attempt_cross_compile: - if build_host: + ensure_build_executor_configured(build_environment_provider, build_executor) + if build_executor: await self._log( "stdout", - f"{icon} Target supports cross compilation; building binary via build host", + f"{icon} Target supports cross compilation; building binary via {build_executor_display_name(build_executor)}", ) else: await self._log("stdout", f"{icon} Target supports cross compilation; building binary locally") @@ -324,13 +353,14 @@ async def build( prebuilt_target=plan.prebuilt_target, target_override=plan.target, logger=self._log, - build_host=build_host, + build_host=build_executor, ) except Exception as exc: error_message = str(exc) failure_msg = f"Cross compilation failed ({exc})" - if build_host: - failure_msg = f"Cross compilation failed on build host ({exc})" + if build_executor: + executor_name = build_executor_kind_name(build_executor) + failure_msg = f"Cross compilation failed on {executor_name} ({exc})" await self._log( "stderr", f"{icon} {failure_msg}", @@ -355,7 +385,7 @@ async def build( "stderr", f"{icon} If the failure is caused by a stale linker cache, clear the build cache (press ... in logs or settings) and try deploying again.", ) - if plan.force_cross_compile: + if plan.force_cross_compile or not plan.allow_on_device_fallback: raise else: await self._log( @@ -365,7 +395,7 @@ async def build( else: cross_compiled = True await self._log("stdout", f"{icon} Cross compilation succeeded; skipping remote build") - elif plan.force_cross_compile: + elif plan.force_cross_compile or not plan.allow_on_device_fallback: raise RuntimeError("Cross compilation required but not supported for this target") return FrameBinaryBuildResult( diff --git a/backend/app/tasks/buildroot_image.py b/backend/app/tasks/buildroot_image.py index 5ab216ff5..5622e506c 100644 --- a/backend/app/tasks/buildroot_image.py +++ b/backend/app/tasks/buildroot_image.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import gzip import copy import hashlib @@ -10,11 +11,12 @@ import shlex import shutil import tempfile +from contextlib import suppress from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache from pathlib import Path -from typing import Any, Optional +from typing import Any, Awaitable, Literal, Optional, TypeVar from types import SimpleNamespace from urllib.parse import urljoin @@ -52,9 +54,18 @@ ) from app.tasks.utils import get_fresh_frame from app.tasks.prebuilt_deps import resolve_prebuilt_target +from app.utils.build_environment import selected_build_environment_provider +from app.utils.build_host import BuildHostConfig, get_build_executor_config +from app.utils.build_executor import ( + BuildExecutor, + DockerMount, + build_executor_display_name, + create_build_executor, + ensure_build_executor_configured, +) from app.utils.cross_compile import CrossCompiler, TargetMetadata +from app.utils.modal_sandbox import ModalSandboxConfig from app.utils.ssh_key_utils import select_ssh_keys_for_frame -from app.utils.local_exec import exec_local_command from app.utils.token import secure_token from app.utils.versions import current_frameos_version @@ -87,6 +98,7 @@ BUILDROOT_BASE_USE_REMOTE = os.environ.get("FRAMEOS_BUILDROOT_BASE_USE_REMOTE", "").lower() in {"1", "true", "yes"} BUILDROOT_BASE_TIMEOUT = float(os.environ.get("FRAMEOS_BUILDROOT_BASE_TIMEOUT", "60")) BUILDROOT_DOCKER_NOFILE_LIMIT = int(os.environ.get("FRAMEOS_BUILDROOT_DOCKER_NOFILE_LIMIT", "65535")) +BUILDROOT_PROGRESS_LOG_INTERVAL_SECONDS = float(os.environ.get("FRAMEOS_BUILDROOT_PROGRESS_LOG_INTERVAL_SECONDS", "30")) BUILDROOT_DOCKER_APT_DEPS = ( "bc", "bison", @@ -173,6 +185,7 @@ JobStatus.queued, JobStatus.in_progress, } +T = TypeVar("T") def normalize_buildroot_platform(platform: str | None) -> str: @@ -728,8 +741,53 @@ def __init__(self, *, db: Session, redis: Redis, frame: Frame, request_id: str | self.redis = redis self.frame = frame self.request_id = request_id + self.build_executor_config: BuildHostConfig | ModalSandboxConfig | None = None + self.executor: BuildExecutor | None = None async def run(self) -> dict[str, Any]: + settings = _get_frame_settings(self.db, self.frame) + provider = selected_build_environment_provider(settings) + self.build_executor_config = self._selected_build_executor() + if provider == "none": + raise RuntimeError( + "Buildroot SD image generation requires Docker, build host, or Modal sandboxes as the global build environment." + ) + ensure_build_executor_configured(provider, self.build_executor_config) + + executor = create_build_executor( + self.build_executor_config, + db=self.db, + redis=self.redis, + frame=self.frame, + logger=self._log, + workspace_prefix="frameos-buildroot-", + ) + if self.build_executor_config: + connection_action = ( + f"Connecting to {build_executor_display_name(self.build_executor_config)} for Buildroot SD image generation" + if executor.connects_on_enter + else ( + f"Using {build_executor_display_name(self.build_executor_config)} for Buildroot SD image generation; " + "sandbox will be created when the build command starts" + ) + ) + await self._log( + "stdout", + connection_action, + ) + async with executor: + self.executor = executor + if self.build_executor_config and executor.connects_on_enter: + await self._log( + "stdout", + f"Connected to {build_executor_display_name(self.build_executor_config)} for Buildroot SD image generation", + ) + try: + return await self._run_with_context() + finally: + self.executor = None + + async def _run_with_context(self) -> dict[str, Any]: validate_buildroot_wifi_credentials(self.frame) bootstrap_frame = self._buildroot_bootstrap_frame() setup_payload = _buildroot_setup_payload(self.db, self.frame) @@ -772,7 +830,7 @@ async def run(self) -> dict[str, Any]: await self._log("stdout", f"Starting Buildroot SD image build {build_id}") base_entry = await resolve_buildroot_base_entry(SUPPORTED_BUILDROOT_PLATFORM) - compose_image = None if self._host_has_compose_tools() else await self._ensure_buildroot_image() + compose_image = await self._compose_tools_image() precompiled_sd_image = await self._try_compose_precompiled_sd_image( temp_dir=temp_dir, output_path=raw_output_path, @@ -817,9 +875,19 @@ async def run(self) -> dict[str, Any]: raise RuntimeError(f"SD image composer completed without producing {raw_output_path.name}") raw_size = raw_output_path.stat().st_size - raw_sha256 = _sha256(raw_output_path) - _gzip_file(raw_output_path, output_path) + raw_sha256 = await self._with_progress_updates( + "Still checksumming raw Buildroot SD image", + asyncio.to_thread(_sha256, raw_output_path), + ) + await self._with_progress_updates( + "Still compressing Buildroot SD image", + asyncio.to_thread(_gzip_file, raw_output_path, output_path), + ) raw_output_path.unlink(missing_ok=True) + compressed_sha256 = await self._with_progress_updates( + "Still checksumming compressed Buildroot SD image", + asyncio.to_thread(_sha256, output_path), + ) metadata = { **_preserved_queue_metadata(latest_buildroot_sd_image(self.frame) or {}), @@ -854,7 +922,7 @@ async def run(self) -> dict[str, Any]: "rawSize": raw_size, "rawSha256": raw_sha256, "size": output_path.stat().st_size, - "sha256": _sha256(output_path), + "sha256": compressed_sha256, "downloadUrl": f"/api/projects/{self.frame.project_id}/frames/{self.frame.id}/buildroot/sd_image/download", "createdAt": _utc_now(), "completedAt": _utc_now(), @@ -869,6 +937,36 @@ def _can_use_precompiled_sd_image(self) -> bool: and frame_compiled_scene_count(self.frame) == 0 ) + def _selected_build_executor(self) -> BuildHostConfig | ModalSandboxConfig | None: + project_id = getattr(self.frame, "project_id", None) + if self.db is None or project_id is None: + return None + return get_build_executor_config(self.db, project_id) + + async def _compose_tools_image(self) -> str | None: + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + if not self.executor.uses_local_filesystem: + return await self._ensure_buildroot_image() + return None if self._host_has_compose_tools() else await self._ensure_buildroot_image() + + async def _run_command( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> tuple[int, str | None, str | None]: + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + return await self.executor.run( + command, + log_command=log_command, + log_output=log_output, + stderr_log_tag=stderr_log_tag, + ) + async def _try_compose_precompiled_sd_image( self, *, @@ -927,6 +1025,7 @@ async def _build_frameos_binary( ) plan = await builder.plan_build( force_cross_compile=False, + allow_on_device_fallback=False, target_override=FRAMEOS_BUILD_TARGET, compilation_mode=frame_compilation_mode(frame), ) @@ -973,6 +1072,7 @@ async def _build_agent_binary(self, deployer: FrameDeployer, temp_dir: str, fram output_name="frameos_agent", compile_script_name="compile_frameos_agent.sh", needs_quickjs=False, + build_host=self.build_executor_config, ).build(source_dir) def _stage_overlay( @@ -1500,27 +1600,28 @@ async def _run_buildroot( skip_apt_install: bool, ) -> None: await self._log("stdout", f"Running Buildroot {BUILDROOT_VERSION} for Raspberry Pi Zero 2 W") - docker_cmd = " ".join( - [ - "docker run --rm", - f"--ulimit nofile={BUILDROOT_DOCKER_NOFILE_LIMIT}:{BUILDROOT_DOCKER_NOFILE_LIMIT}", - f"-v {shlex.quote(str(temp_dir))}:/work", - f"-v {shlex.quote(str(source_dir))}:/build/buildroot", - f"-v {shlex.quote(str(output_dir))}:/build/output", - f"-v {shlex.quote(str(cache_dir))}:/cache", - f"-v {shlex.quote(str(artifact_dir))}:/artifacts", - *(["-e BUILDROOT_SKIP_APT_INSTALL=1"] if skip_apt_install else []), - "-e FORCE_UNSAFE_CONFIGURE=1", - shlex.quote(image), - "bash /work/buildroot-build.sh", - ] - ) - status, _, err = await exec_local_command( - self.db, - self.redis, - self.frame, - docker_cmd, - log_command="docker run (buildroot image)", + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + status, _, err = await self._with_progress_updates( + "Still running Buildroot image build", + self.executor.docker_run( + image=image, + mounts=[ + DockerMount(temp_dir, "/work"), + DockerMount(source_dir, "/build/buildroot"), + DockerMount(output_dir, "/build/output"), + DockerMount(cache_dir, "/cache"), + DockerMount(artifact_dir, "/artifacts"), + ], + env={ + **({"BUILDROOT_SKIP_APT_INSTALL": "1"} if skip_apt_install else {}), + "FORCE_UNSAFE_CONFIGURE": "1", + }, + ulimits=[f"nofile={BUILDROOT_DOCKER_NOFILE_LIMIT}:{BUILDROOT_DOCKER_NOFILE_LIMIT}"], + args=["bash", "/work/buildroot-build.sh"], + workspace="buildroot-image", + log_command="docker run (buildroot image)", + ), ) if status != 0: raise RuntimeError(f"Buildroot image build failed: {err or 'see logs'}") @@ -1601,16 +1702,19 @@ async def _compose_sd_image_from_base( (compose_dir / "empty-root").mkdir(exist_ok=True) if image: - docker_work_dir = compose_dir.resolve() - compose_cmd = " ".join( - [ - "docker run --rm", - f"-v {shlex.quote(str(docker_work_dir))}:/work", - shlex.quote(image), - "bash /work/compose-partitions.sh", - ] + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + status, _, err = await self._with_progress_updates( + "Still composing Buildroot SD image partitions", + self.executor.docker_run( + image=image, + mounts=[DockerMount(compose_dir.resolve(), "/work")], + args=["bash", "/work/compose-partitions.sh"], + workspace="compose", + log_command="docker run (buildroot image composer)", + stderr_log_tag="stdout", + ), ) - log_command = "docker run (buildroot image composer)" else: compose_cmd = " ".join( [ @@ -1621,28 +1725,47 @@ async def _compose_sd_image_from_base( ] ) log_command = "buildroot image composer" - status, _, err = await exec_local_command( - self.db, - self.redis, - self.frame, - compose_cmd, - log_command=log_command, - stderr_log_tag="stdout", - ) + status, _, err = await self._with_progress_updates( + "Still composing Buildroot SD image partitions", + self._run_command( + compose_cmd, + log_command=log_command, + stderr_log_tag="stdout", + ), + ) if status != 0: raise RuntimeError(f"Buildroot image composition failed: {err or 'see logs'}") + partitions = await self._with_progress_updates( + "Still applying composed partitions to Buildroot SD image", + asyncio.to_thread( + self._apply_composed_partitions, + base_image_path, + output_path, + images_dir / "frameos.ext4", + images_dir / "assets.vfat", + ), + ) + await self._patch_boot_partition(output_path, partitions, boot_root, image=image) + + @staticmethod + def _apply_composed_partitions( + base_image_path: Path, + output_path: Path, + frameos_image: Path, + assets_image: Path, + ) -> list[dict[str, int]]: shutil.copy2(base_image_path, output_path) partitions = _mbr_partitions(output_path) partitions = _shrink_data_partitions( output_path, partitions, - frameos_image=images_dir / "frameos.ext4", - assets_image=images_dir / "assets.vfat", + frameos_image=frameos_image, + assets_image=assets_image, ) - _replace_partition(output_path, partitions, 3, images_dir / "frameos.ext4") - _replace_partition(output_path, partitions, 4, images_dir / "assets.vfat") - await self._patch_boot_partition(output_path, partitions, boot_root, image=image) + _replace_partition(output_path, partitions, 3, frameos_image) + _replace_partition(output_path, partitions, 4, assets_image) + return partitions async def _compose_sd_image_from_precompiled_release( self, @@ -1662,6 +1785,18 @@ async def _compose_sd_image_from_precompiled_release( shutil.rmtree(boot_root) shutil.copytree(temp_dir / "overlay" / "boot", boot_root, symlinks=True) + partitions = await self._with_progress_updates( + "Still customizing precompiled Buildroot SD image", + asyncio.to_thread( + self._prepare_precompiled_release_image, + release_image_path, + output_path, + ), + ) + await self._patch_boot_partition(output_path, partitions, boot_root, image=image) + + @staticmethod + def _prepare_precompiled_release_image(release_image_path: Path, output_path: Path) -> list[dict[str, int]]: if release_image_path.name.endswith(".gz"): _gunzip_file(release_image_path, output_path) else: @@ -1674,7 +1809,7 @@ async def _compose_sd_image_from_precompiled_release( raise RuntimeError( "Full precompiled Buildroot SD image uses larger data partitions than the current image layout" ) - await self._patch_boot_partition(output_path, partitions, boot_root, image=image) + return partitions async def _patch_boot_partition( self, @@ -1798,20 +1933,23 @@ async def _patch_boot_partition( os.chmod(script_path, 0o755) if image: - docker_image_dir = output_path.parent.resolve() - docker_boot_root = boot_root.resolve() - docker_script_path = script_path.resolve() - patch_cmd = " ".join( - [ - "docker run --rm", - f"-v {shlex.quote(str(docker_image_dir))}:/image", - f"-v {shlex.quote(str(docker_boot_root))}:/boot-root", - f"-v {shlex.quote(str(docker_script_path))}:/patch-boot.sh", - shlex.quote(image), - "bash /patch-boot.sh", - ] + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + status, _, err = await self._with_progress_updates( + "Still patching Buildroot SD image boot partition", + self.executor.docker_run( + image=image, + mounts=[ + DockerMount(output_path.resolve(), f"/image/{output_path.name}"), + DockerMount(boot_root.resolve(), "/boot-root"), + DockerMount(script_path.resolve(), "/patch-boot.sh", read_only=True), + ], + args=["bash", "/patch-boot.sh"], + workspace="boot-patch", + log_command="docker run (buildroot boot partition patch)", + stderr_log_tag="stdout", + ), ) - log_command = "docker run (buildroot boot partition patch)" else: patch_cmd = " ".join( [ @@ -1822,20 +1960,48 @@ async def _patch_boot_partition( ] ) log_command = "buildroot boot partition patch" - status, _, err = await exec_local_command( - self.db, - self.redis, - self.frame, - patch_cmd, - log_command=log_command, - stderr_log_tag="stdout", - ) + status, _, err = await self._with_progress_updates( + "Still patching Buildroot SD image boot partition", + self._run_command( + patch_cmd, + log_command=log_command, + stderr_log_tag="stdout", + ), + ) if status != 0: raise RuntimeError(f"Buildroot BOOT partition patch failed: {err or 'see logs'}") async def _log(self, type: str, line: str) -> None: await log(self.db, self.redis, int(self.frame.id), type, line) + async def _with_progress_updates(self, message: str, awaitable: Awaitable[T]) -> T: + interval = BUILDROOT_PROGRESS_LOG_INTERVAL_SECONDS + if interval <= 0: + return await awaitable + + async def progress_logger() -> None: + elapsed = 0.0 + while True: + await asyncio.sleep(interval) + elapsed += interval + await self._log("stdout", f"{message} ({self._format_elapsed(elapsed)} elapsed)") + + task = asyncio.create_task(progress_logger()) + try: + return await awaitable + finally: + task.cancel() + with suppress(asyncio.CancelledError): + await task + + @staticmethod + def _format_elapsed(seconds: float) -> str: + total_seconds = max(1, int(round(seconds))) + minutes, remainder = divmod(total_seconds, 60) + if minutes: + return f"{minutes}m {remainder:02d}s" + return f"{remainder}s" + @staticmethod def _sanitize(value: str) -> str: return SAFE_SEGMENT.sub("_", value or "unknown") @@ -1872,10 +2038,7 @@ def _resolved_buildroot_image(self) -> str: return image async def _buildroot_image_has_compose_tools(self, image: str) -> bool: - status, _out, _err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _out, _err = await self._run_command( " ".join( [ "docker run --rm", @@ -1901,11 +2064,12 @@ def _host_has_compose_tools() -> bool: async def _ensure_buildroot_image(self) -> str: image = self._buildroot_image() resolved_image = self._resolved_buildroot_image() + if self.executor is None: + raise RuntimeError("Build executor unavailable during Buildroot SD image generation") + if self.executor.uses_container_images_directly: + return self.executor.container_image_reference(image, resolved_image) if not BUILDROOT_FORCE_LOCAL_BUILD: - status, _out, _err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _out, _err = await self._run_command( f"docker image inspect {shlex.quote(resolved_image)} >/dev/null 2>&1", log_command=False, log_output=False, @@ -1914,10 +2078,7 @@ async def _ensure_buildroot_image(self) -> str: return resolved_image if resolved_image != image: - status, _out, _err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _out, _err = await self._run_command( f"docker image inspect {shlex.quote(image)} >/dev/null 2>&1", log_command=False, log_output=False, @@ -1926,10 +2087,7 @@ async def _ensure_buildroot_image(self) -> str: return image legacy_image = self._legacy_buildroot_image() - status, _out, _err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _out, _err = await self._run_command( f"docker image inspect {shlex.quote(legacy_image)} >/dev/null 2>&1", log_command=False, log_output=False, @@ -1939,10 +2097,7 @@ async def _ensure_buildroot_image(self) -> str: if not BUILDROOT_SKIP_PULL: pull_cmd = f"docker pull {shlex.quote(resolved_image)}" - status, _pull_out, pull_err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _pull_out, pull_err = await self._run_command( pull_cmd, log_command=f"docker pull {shlex.quote(resolved_image)}", log_output=False, @@ -1951,10 +2106,7 @@ async def _ensure_buildroot_image(self) -> str: return resolved_image if resolved_image != image: - status, _pull_out, pull_err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _pull_out, pull_err = await self._run_command( f"docker pull {shlex.quote(image)}", log_command=f"docker pull {shlex.quote(image)}", log_output=False, @@ -1972,6 +2124,11 @@ async def _ensure_buildroot_image(self) -> str: "Buildroot Dockerfile is missing; expected at backend/tools/buildroot.Dockerfile", ) + context_dir, dockerfile_arg = await self.executor.prepare_docker_build_context( + BUILDROOT_DOCKERFILE, + "buildroot-dockerfile", + ) + build_cmd = " ".join( [ "docker build --load", @@ -1979,14 +2136,11 @@ async def _ensure_buildroot_image(self) -> str: f"--build-arg BUILDROOT_VERSION={shlex.quote(BUILDROOT_VERSION)}", f"--build-arg BUILDROOT_APT_DEPS={shlex.quote(BUILDROOT_DOCKER_APT_DEPS_LINE)}", f"-t {shlex.quote(image)}", - f"-f {shlex.quote(str(BUILDROOT_DOCKERFILE))}", - shlex.quote(str(BUILDROOT_DOCKERFILE.parent)), + f"-f {shlex.quote(dockerfile_arg)}", + shlex.quote(context_dir), ] ) - status, _stdout, err = await exec_local_command( - self.db, - self.redis, - self.frame, + status, _stdout, err = await self._run_command( build_cmd, log_command="docker build (buildroot image)", ) @@ -2751,20 +2905,10 @@ def _shrink_data_partitions( assets_size = assets_image.stat().st_size frameos_partition = partitions[2] assets_partition = partitions[3] - if frameos_size > frameos_partition["size"]: - raise RuntimeError( - f"{frameos_image.name} is larger than partition 3: {frameos_size} > {frameos_partition['size']}" - ) - if assets_size > assets_partition["size"]: - raise RuntimeError( - f"{assets_image.name} is larger than partition 4: {assets_size} > {assets_partition['size']}" - ) frameos_start = frameos_partition["start"] assets_start = _align_up_bytes(frameos_start + frameos_size) output_size = assets_start + assets_size - if output_size > image_path.stat().st_size: - raise RuntimeError(f"Shrunk data partition layout would exceed {image_path.name}") _set_mbr_partition_geometry(image_path, 3, start=frameos_start, size=frameos_size) _set_mbr_partition_geometry(image_path, 4, start=assets_start, size=assets_size) diff --git a/backend/app/tasks/deploy_agent.py b/backend/app/tasks/deploy_agent.py index 31fcf7851..1193658dc 100644 --- a/backend/app/tasks/deploy_agent.py +++ b/backend/app/tasks/deploy_agent.py @@ -14,13 +14,16 @@ from app.models.frame import Frame from app.models.log import new_log as log +from app.models.settings import get_settings_dict +from app.utils.build_environment import selected_build_environment_provider from app.utils.local_exec import exec_local_command from app.utils.remote_exec import RemoteTransport, upload_file from app.tasks._frame_deployer import FrameDeployer from app.tasks.frame_deploy_helpers import sanitize_apt_package_name from app.tasks.prebuilt_deps import resolve_prebuilt_target from app.tasks.precompiled_agent import download_precompiled_agent_release -from app.utils.build_host import get_build_host_config +from app.utils.build_host import get_build_executor_config +from app.utils.build_executor import build_executor_display_name, ensure_build_executor_configured from app.utils.cross_compile import CrossCompiler, TargetMetadata, can_cross_compile_target from app.utils.versions import current_agent_version, get_versions from .utils import find_nim_v2, find_nimbase_file, get_fresh_frame @@ -32,6 +35,13 @@ REPO_ROOT = Path(__file__).resolve().parents[3] +def get_build_host_config(*args, **kwargs): # noqa: ANN002, ANN003 + """Compatibility shim for older tests monkeypatching this module symbol.""" + from app.utils.build_host import get_build_host_config as _get_build_host_config + + return _get_build_host_config(*args, **kwargs) + + def precompiled_agent_enabled() -> bool: return os.environ.get(PRECOMPILED_AGENT_ENV, "").strip().lower() not in { "0", @@ -366,17 +376,25 @@ async def _try_cross_compile_agent( distro: str, distro_version: str, ) -> bool: + project_id = getattr(self.frame, "project_id", None) + build_executor = get_build_executor_config(self.db, project_id) + settings = get_settings_dict(self.db, project_id=project_id) if self.db and project_id is not None else {} + build_environment_provider = selected_build_environment_provider(settings) + if build_environment_provider == "none": + await self.log("stdout", "- Server-side compilation is disabled; building agent on the device") + return False if not can_cross_compile_target(arch): + if build_environment_provider != "none": + raise RuntimeError(f"Selected build environment cannot cross compile agent for {arch}") await self.log( "stdout", f"- Agent target architecture {arch} does not support cross compilation; building on the device", ) return False - project_id = getattr(self.frame, "project_id", None) - build_host = get_build_host_config(self.db, project_id) - if build_host: - await self.log("stdout", "- Cross compiling agent via build host") + ensure_build_executor_configured(build_environment_provider, build_executor) + if build_executor: + await self.log("stdout", f"- Cross compiling agent via {build_executor_display_name(build_executor)}") else: await self.log("stdout", "- Cross compiling agent locally") @@ -390,12 +408,15 @@ async def _try_cross_compile_agent( temp_dir=self.temp_dir, build_dir=build_dir, logger=self.log, - build_host=build_host, + build_host=build_executor, output_name=AGENT_BINARY, compile_script_name="compile_frameos_agent.sh", needs_quickjs=False, ).build(source_dir) except Exception as exc: # noqa: BLE001 + if build_environment_provider != "none": + await self.log("stderr", f"- Agent cross compilation failed ({exc})") + raise await self.log("stderr", f"- Agent cross compilation failed ({exc}); falling back to on-device build") return False diff --git a/backend/app/tasks/frame_deploy_workflow.py b/backend/app/tasks/frame_deploy_workflow.py index 64a9e8b56..d73f35828 100644 --- a/backend/app/tasks/frame_deploy_workflow.py +++ b/backend/app/tasks/frame_deploy_workflow.py @@ -37,6 +37,7 @@ render_setup_json_reset_service, setup_json_reset_file_path, ) +from app.utils.build_environment import selected_build_environment_provider from app.utils.frame_http import _fetch_frame_http_bytes from app.utils.remote_exec import upload_file from app.utils.ssh_authorized_keys import _install_authorized_keys @@ -394,6 +395,8 @@ async def _plan_full(self, *, frame_dict: dict[str, Any], previous_frameos_versi drivers = drivers_for_frame(self.frame) driver_names = sorted(drivers.keys()) + settings = _get_frame_settings(self.db, self.frame) + build_environment_provider = selected_build_environment_provider(settings) compile_settings = (self.frame.buildroot if is_buildroot else self.frame.rpios) or {} cross_compilation_setting = ( "always" if is_buildroot else (compile_settings.get("crossCompilation") or "auto").lower() @@ -402,15 +405,22 @@ async def _plan_full(self, *, frame_dict: dict[str, Any], previous_frameos_versi cross_compilation_setting = "auto" compilation_mode = normalize_compilation_mode(compile_settings.get("compilationMode")) - allow_cross_compile = cross_compilation_setting != "never" + if build_environment_provider == "none" and (is_buildroot or cross_compilation_setting == "always"): + raise RuntimeError( + "Server-side compilation is disabled in global settings. Choose Docker, build host, or Modal " + "sandboxes as the build environment, or set this frame to always compile on device." + ) + + allow_cross_compile = cross_compilation_setting != "never" and build_environment_provider != "none" force_cross_compile = is_buildroot or cross_compilation_setting == "always" + allow_on_device_fallback = build_environment_provider == "none" or cross_compilation_setting == "never" binary_plan = await self.binary_builder.plan_build( allow_cross_compile=allow_cross_compile, force_cross_compile=force_cross_compile, + allow_on_device_fallback=allow_on_device_fallback, compilation_mode=compilation_mode, ) - settings = _get_frame_settings(self.db, self.frame) selected_keys = select_ssh_keys_for_frame(self.frame, settings) selected_public_keys = [key.get("public") for key in selected_keys if key.get("public")] known_public_keys = [key.get("public") for key in normalize_ssh_keys(settings) if key.get("public")] @@ -1219,8 +1229,26 @@ async def _install_setup_json_reset_helper(self, setup_json_reset_path: str) -> await self.deployer.exec_command(f"sudo systemctl enable {SETUP_JSON_RESET_SERVICE_NAME}", raise_on_error=False) async def _remove_setup_json_reset_helper(self) -> None: + status = await self.deployer.exec_command( + "test -e /etc/systemd/system/frameos-firstboot-setup.service " + "|| test -e /usr/local/bin/frameos-setup-reset.sh " + "|| test -e /srv/frameos/build/frameos-firstboot-setup.service " + "|| test -e /srv/frameos/build/frameos-setup-reset.sh", + log_command=False, + log_output=False, + raise_on_error=False, + ) + if status != 0: + return + await self.deployer.log("stdout", f"{icon} Removing setup JSON reset helper") - await self.deployer.exec_command(f"sudo systemctl disable {SETUP_JSON_RESET_SERVICE_NAME}", raise_on_error=False) + await self.deployer.exec_command( + f"if systemctl list-unit-files --type=service --no-legend {SETUP_JSON_RESET_SERVICE_NAME} " + f"| grep -q '^{SETUP_JSON_RESET_SERVICE_NAME}'; then " + f"sudo systemctl disable {SETUP_JSON_RESET_SERVICE_NAME}; " + "fi", + raise_on_error=False, + ) await self._exec_host_root_command( f"rm -f {shlex.quote(SETUP_JSON_RESET_SERVICE_PATH)} {shlex.quote(SETUP_JSON_RESET_SCRIPT_PATH)}", fallback_command=f"sudo rm -f {shlex.quote(SETUP_JSON_RESET_SERVICE_PATH)} {shlex.quote(SETUP_JSON_RESET_SCRIPT_PATH)}", diff --git a/backend/app/tasks/tests/test_buildroot_image.py b/backend/app/tasks/tests/test_buildroot_image.py index 023a45707..777ff93bf 100644 --- a/backend/app/tasks/tests/test_buildroot_image.py +++ b/backend/app/tasks/tests/test_buildroot_image.py @@ -1,11 +1,12 @@ from __future__ import annotations +import asyncio import gzip import importlib.util import json import re import sys -from pathlib import Path +from pathlib import Path, PurePosixPath from types import SimpleNamespace import pytest @@ -35,12 +36,34 @@ render_setup_json_reset_script, setup_json_reset_file_path, ) +from app.utils.build_executor import BuildHostExecutor +from app.utils.build_host import BuildHostConfig from app.utils.cross_compile import CrossCompiler REPO_ROOT = Path(__file__).resolve().parents[4] +@pytest.mark.asyncio +async def test_buildroot_progress_updates_after_interval(monkeypatch): + monkeypatch.setattr(buildroot_image_module, "BUILDROOT_PROGRESS_LOG_INTERVAL_SECONDS", 0.01) + builder = BuildrootImageBuilder(db=None, redis=None, frame=SimpleNamespace(id=1)) + logs: list[tuple[str, str]] = [] + + async def fake_log(type: str, line: str) -> None: + logs.append((type, line)) + + builder._log = fake_log + + result = await builder._with_progress_updates("Still working on SD image", asyncio.sleep(0.025, result="done")) + + assert result == "done" + assert logs + assert logs[0][0] == "stdout" + assert logs[0][1].startswith("Still working on SD image (") + assert "elapsed" in logs[0][1] + + def test_buildroot_frameos_cross_target_uses_docker_arm64_platform(tmp_path, monkeypatch): monkeypatch.setenv("FRAMEOS_CROSS_CACHE", str(tmp_path / "cross-cache")) @@ -91,6 +114,43 @@ def test_buildroot_firstboot_setup_uses_with_setup_command(): assert "--from-file" not in script +@pytest.mark.asyncio +async def test_buildroot_frameos_binary_disables_on_device_fallback(monkeypatch, tmp_path): + calls = {} + frame = SimpleNamespace(id=1, project_id=1, rpios={}) + + class FakeFrameBinaryBuilder: + def __init__(self, **kwargs): + calls["init"] = kwargs + + async def plan_build(self, **kwargs): + calls["plan"] = kwargs + return "plan" + + async def build(self, plan, **kwargs): + calls["build"] = {"plan": plan, **kwargs} + return "result" + + monkeypatch.setattr(buildroot_image_module, "FrameBinaryBuilder", FakeFrameBinaryBuilder) + + async def fake_log(*_args, **_kwargs): + return None + + builder = BuildrootImageBuilder(db=object(), redis=object(), frame=frame) + monkeypatch.setattr(builder, "_log", fake_log) + result = await builder._build_frameos_binary( + SimpleNamespace(build_id="build12345678"), + str(tmp_path), + frame, + ) + + assert result == "result" + assert calls["plan"]["target_override"] == FRAMEOS_BUILD_TARGET + assert calls["plan"]["allow_on_device_fallback"] is False + assert calls["build"]["plan"] == "plan" + assert calls["build"]["precompiled_install_all_drivers"] is True + + def test_buildroot_defaults_remove_setup_json_reset_file_path(): frame = SimpleNamespace( id=7, @@ -422,13 +482,15 @@ async def fake_download_precompiled_buildroot_sd_image(**_kwargs): ) async def fake_exec_local_command(*args, **kwargs): - command = args[3] + command = args[0] commands.append(command) captured["patch_script"] = (tmp_path / "tmp" / "precompiled-compose" / "patch-boot.sh").read_text( encoding="utf-8" ) return 0, "", "" + builder.executor = SimpleNamespace(run=fake_exec_local_command) + async def fake_log(level, message): logs.append((level, message)) @@ -441,7 +503,6 @@ def fail_replace_partition(*_args, **_kwargs): "app.tasks.buildroot_image.download_precompiled_buildroot_sd_image", fake_download_precompiled_buildroot_sd_image, ) - monkeypatch.setattr("app.tasks.buildroot_image.exec_local_command", fake_exec_local_command) monkeypatch.setattr( "app.tasks.buildroot_image._mbr_partitions", lambda _path: [ @@ -521,6 +582,38 @@ def test_shrink_data_partitions_rewrites_mbr_and_truncates_image(tmp_path): assert image.stat().st_size == shrunk[3]["start"] + shrunk[3]["size"] +def test_shrink_data_partitions_can_grow_trailing_data_partitions(tmp_path): + image = tmp_path / "base.img" + frameos = tmp_path / "frameos.ext4" + assets = tmp_path / "assets.vfat" + partitions = [ + (512, 32 * 1024 * 1024), + (32 * 1024 * 1024 + 512, 160 * 1024 * 1024), + (192 * 1024 * 1024 + 512, 30 * 1024 * 1024), + (222 * 1024 * 1024 + 512, 30 * 1024 * 1024), + ] + _write_test_mbr(image, partitions) + with image.open("r+b") as image_file: + image_file.truncate(partitions[-1][0] + partitions[-1][1]) + frameos.write_bytes(b"\0" * (70 * 1024 * 1024)) + assets.write_bytes(b"\0" * (30 * 1024 * 1024)) + + grown = buildroot_image_module._shrink_data_partitions( + image, + buildroot_image_module._mbr_partitions(image), + frameos_image=frameos, + assets_image=assets, + ) + + assert grown[2] == {"start": partitions[2][0], "size": 70 * 1024 * 1024} + expected_assets_start = buildroot_image_module._align_up_bytes(partitions[2][0] + 70 * 1024 * 1024) + assert grown[3] == { + "start": expected_assets_start, + "size": 30 * 1024 * 1024, + } + assert image.stat().st_size == grown[3]["start"] + grown[3]["size"] + + def test_partition_size_for_root_grows_with_payload_size(tmp_path): root = tmp_path / "root" root.mkdir() @@ -577,14 +670,14 @@ async def test_buildroot_docker_run_raises_nofile_limit(tmp_path, monkeypatch): builder = BuildrootImageBuilder(db=object(), redis=None, frame=SimpleNamespace(id=1)) captured = {} - async def fake_exec_local_command(*args, **kwargs): - captured["command"] = args[3] + async def fake_docker_run(**kwargs): + captured.update(kwargs) return 0, "", "" async def fake_log(*args, **kwargs): return None - monkeypatch.setattr("app.tasks.buildroot_image.exec_local_command", fake_exec_local_command) + builder.executor = SimpleNamespace(docker_run=fake_docker_run) monkeypatch.setattr(builder, "_log", fake_log) await builder._run_buildroot( @@ -597,7 +690,34 @@ async def fake_log(*args, **kwargs): skip_apt_install=True, ) - assert "--ulimit nofile=65535:65535" in captured["command"] + assert captured["ulimits"] == ["nofile=65535:65535"] + + +@pytest.mark.asyncio +async def test_buildroot_image_selection_allows_build_host(monkeypatch): + commands: list[str] = [] + + class FakeExecutor: + uses_container_images_directly = False + + async def run(self, command, **_kwargs): + commands.append(command) + return 0, "", "" + + frame = SimpleNamespace(id=1, project_id=7) + builder = BuildrootImageBuilder(db=object(), redis=None, frame=frame) + builder.executor = FakeExecutor() + + monkeypatch.setattr( + "app.tasks.buildroot_image.get_settings_dict", + lambda _db, project_id=None: {"buildEnvironment": {"provider": "buildHost"}}, + ) + + image = await builder._ensure_buildroot_image() + + assert image + assert any(command.startswith("docker image inspect ") for command in commands) + assert any("command -v genimage" in command for command in commands) @pytest.mark.asyncio @@ -625,18 +745,17 @@ async def test_cached_base_composer_uses_container_visible_srcpaths(tmp_path, mo replaced: list[tuple[int, str]] = [] builder = BuildrootImageBuilder(db=object(), redis=None, frame=SimpleNamespace(id=1)) - async def fake_exec_local_command(*args, **kwargs): - command = args[3] - commands.append(command) - if "compose-partitions.sh" in command: - captured["compose_command"] = command + async def fake_docker_run(**kwargs): + commands.append(kwargs["workspace"]) + if kwargs["workspace"] == "compose": + captured["compose"] = kwargs config = temp_dir / "compose" / "frameos-genimage.cfg" captured["config"] = config.read_text(encoding="utf-8") images_dir = temp_dir / "compose" / "images" (images_dir / "frameos.ext4").write_bytes(b"frameos") (images_dir / "assets.vfat").write_bytes(b"assets") - if "patch-boot.sh" in command: - captured["patch_command"] = command + if kwargs["workspace"] == "boot-patch": + captured["patch"] = kwargs captured["patch_script"] = (temp_dir / "compose" / "patch-boot.sh").read_text(encoding="utf-8") return 0, "", "" @@ -649,7 +768,7 @@ def fake_replace_partition(_image_path, _partitions, partition_number, partition def fake_shrink_data_partitions(_image_path, partitions, **_kwargs): return partitions - monkeypatch.setattr("app.tasks.buildroot_image.exec_local_command", fake_exec_local_command) + builder.executor = SimpleNamespace(docker_run=fake_docker_run) monkeypatch.setattr( "app.tasks.buildroot_image._partition_size_for_root", lambda _root, *, minimum_size: "42M", @@ -683,12 +802,13 @@ def fake_shrink_data_partitions(_image_path, partitions, **_kwargs): assert "size = 42M" in captured["config"] assert 'label = "BOOT"' not in captured["config"] assert str(temp_dir) not in captured["config"] - assert "bash /work/compose-partitions.sh" in captured["compose_command"] - assert "bash /patch-boot.sh" in captured["patch_command"] - assert f"-v {tmp_path / 'release-assets'}:/image" in captured["patch_command"] - assert "-v release-assets:/image" not in captured["patch_command"] - assert "frameos/frameos-buildroot:test" in captured["compose_command"] - assert "frameos/frameos-buildroot:test" in captured["patch_command"] + assert captured["compose"]["args"] == ["bash", "/work/compose-partitions.sh"] + assert captured["patch"]["args"] == ["bash", "/patch-boot.sh"] + patch_mounts = captured["patch"]["mounts"] + assert patch_mounts[0].source == output_image.resolve() + assert patch_mounts[0].target == "/image/output.img" + assert captured["compose"]["image"] == "frameos/frameos-buildroot:test" + assert captured["patch"]["image"] == "frameos/frameos-buildroot:test" assert 'tar -C "$work_dir/roots" -cf - frameos assets | tar -C "$compose_roots" -xf -' in ( temp_dir / "compose" / "compose-partitions.sh" ).read_text(encoding="utf-8") @@ -701,6 +821,112 @@ def fake_shrink_data_partitions(_image_path, partitions, **_kwargs): assert len(commands) == 2 +@pytest.mark.asyncio +async def test_cached_base_composer_runs_docker_on_build_host_paths(tmp_path, monkeypatch): + temp_dir = tmp_path / "tmp" + frameos_overlay = temp_dir / "overlay" / "srv" / "frameos" + assets_overlay = temp_dir / "overlay" / "srv" / "assets" + boot_overlay = temp_dir / "overlay" / "boot" + frameos_overlay.mkdir(parents=True) + assets_overlay.mkdir(parents=True) + boot_overlay.mkdir(parents=True) + (frameos_overlay / "frameos").write_bytes(b"binary") + (boot_overlay / "frameos-setup.json").write_text("{}", encoding="utf-8") + base_image = tmp_path / "base.img" + output_image = tmp_path / "output.img" + base_image.write_bytes(b"base") + commands: list[str] = [] + synced_dirs: list[tuple[str, str]] = [] + downloaded_dirs: list[tuple[str, str]] = [] + synced_files: list[tuple[str, str]] = [] + downloaded_files: list[tuple[str, str]] = [] + replaced: list[tuple[int, str]] = [] + + class FakeBuildHostSession: + async def sync_dir_tarball(self, local_path, remote_path): + synced_dirs.append((local_path, remote_path)) + + async def download_dir_tarball(self, remote_path, local_path): + downloaded_dirs.append((remote_path, local_path)) + local = Path(local_path) + local.mkdir(parents=True, exist_ok=True) + images = local / "images" + images.mkdir(exist_ok=True) + (images / "frameos.ext4").write_bytes(b"frameos") + (images / "assets.vfat").write_bytes(b"assets") + + async def run(self, command, **_kwargs): + commands.append(command) + return 0, "", "" + + async def remove_path(self, _remote_path): + return None + + async def ensure_dir(self, _remote_path): + return None + + async def sync_file(self, local_path, remote_path): + synced_files.append((local_path, remote_path)) + + async def download_file(self, remote_path, local_path): + downloaded_files.append((remote_path, local_path)) + + builder = BuildrootImageBuilder(db=object(), redis=None, frame=SimpleNamespace(id=1)) + executor = BuildHostExecutor( + BuildHostConfig(host="builder.local", user="ubuntu", ssh_key="dummy-key") + ) + executor.session = FakeBuildHostSession() + executor.remote_root = PurePosixPath("/tmp/frameos-buildroot-test") + builder.executor = executor + + async def fake_log(*args, **kwargs): + return None + + def fake_replace_partition(_image_path, _partitions, partition_number, partition_image): + replaced.append((partition_number, partition_image.name)) + + def fake_shrink_data_partitions(_image_path, partitions, **_kwargs): + return partitions + + monkeypatch.setattr( + "app.tasks.buildroot_image._mbr_partitions", + lambda _path: [ + {"start": 512, "size": 32 * 1024 * 1024}, + {"start": 33554944, "size": 768 * 1024 * 1024}, + {"start": 838861312, "size": 512 * 1024 * 1024}, + {"start": 1375732224, "size": 512 * 1024 * 1024}, + ], + ) + monkeypatch.setattr("app.tasks.buildroot_image._shrink_data_partitions", fake_shrink_data_partitions) + monkeypatch.setattr("app.tasks.buildroot_image._replace_partition", fake_replace_partition) + monkeypatch.setattr(builder, "_log", fake_log) + + await builder._compose_sd_image_from_base( + temp_dir=temp_dir, + base_image_path=base_image, + output_path=output_image, + image="frameos/frameos-buildroot:test", + ) + + assert "-v /tmp/frameos-buildroot-test/compose/mount-0-compose:/work" in commands[0] + assert str(temp_dir / "compose") not in commands[0] + assert synced_dirs[0] == ( + str(temp_dir / "compose"), + "/tmp/frameos-buildroot-test/compose/mount-0-compose", + ) + assert ( + "/tmp/frameos-buildroot-test/compose/mount-0-compose", + str(temp_dir / "compose"), + ) in downloaded_dirs + assert "-v /tmp/frameos-buildroot-test/boot-patch/mount-0-output.img:/image/output.img" in commands[1] + assert any(local.endswith("output.img") and remote.endswith("/boot-patch/mount-0-output.img") for local, remote in synced_files) + assert any( + local.endswith("patch-boot.sh") and remote.endswith("/boot-patch/mount-2-patch-boot.sh") + for local, remote in synced_files + ) + assert replaced == [(3, "frameos.ext4"), (4, "assets.vfat")] + + @pytest.mark.asyncio async def test_cached_base_composer_runs_without_docker_when_image_is_not_required(tmp_path, monkeypatch): temp_dir = tmp_path / "tmp" @@ -720,7 +946,7 @@ async def test_cached_base_composer_runs_without_docker_when_image_is_not_requir builder = BuildrootImageBuilder(db=object(), redis=None, frame=SimpleNamespace(id=1)) async def fake_exec_local_command(*args, **kwargs): - command = args[3] + command = args[0] commands.append(command) if "compose-partitions.sh" in command: images_dir = temp_dir / "compose" / "images" @@ -737,7 +963,7 @@ def fake_replace_partition(_image_path, _partitions, partition_number, partition def fake_shrink_data_partitions(_image_path, partitions, **_kwargs): return partitions - monkeypatch.setattr("app.tasks.buildroot_image.exec_local_command", fake_exec_local_command) + builder.executor = SimpleNamespace(run=fake_exec_local_command) monkeypatch.setattr( "app.tasks.buildroot_image._mbr_partitions", lambda _path: [ diff --git a/backend/app/tasks/tests/test_cross_compile.py b/backend/app/tasks/tests/test_cross_compile.py index c0f77e3e1..6c719c95a 100644 --- a/backend/app/tasks/tests/test_cross_compile.py +++ b/backend/app/tasks/tests/test_cross_compile.py @@ -1,11 +1,14 @@ from __future__ import annotations +from pathlib import Path from types import SimpleNamespace import pytest from app.tasks.prebuilt_deps import PrebuiltEntry, resolve_prebuilt_target +from app.utils.build_executor import create_build_executor from app.utils.cross_compile import CrossCompiler, TargetMetadata +from app.utils.modal_sandbox import ModalSandboxConfig def make_cross_compiler( @@ -99,6 +102,31 @@ def test_cross_compiler_canonicalizes_ubuntu_codename(tmp_path, monkeypatch: pyt assert compiler._docker_image() == "ubuntu:24.04" +def test_cross_compiler_bakes_target_cross_packages_into_amd64_toolchain_image( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setenv("FRAMEOS_CROSS_CACHE", str(tmp_path / "cross-cache")) + compiler = CrossCompiler( + db=None, + redis=None, + frame=SimpleNamespace(id=1), + deployer=SimpleNamespace(build_id="build12345678"), + target=TargetMetadata(arch="aarch64", distro="raspios", version="bookworm"), + temp_dir=str(tmp_path / "tmp"), + prebuilt_entry=None, + ) + + dpkg_archs, packages = compiler._target_cross_toolchain_build_args("linux/amd64") + + assert dpkg_archs == "arm64 armhf" + assert "gcc-aarch64-linux-gnu" in packages + assert "libssl-dev:arm64" in packages + assert "gcc-arm-linux-gnueabihf" in packages + assert "libssl-dev:armhf" in packages + assert compiler._target_cross_toolchain_build_args("linux/arm64") == ("", "") + + @pytest.mark.asyncio @pytest.mark.parametrize( ("component", "version"), @@ -176,18 +204,151 @@ async def test_run_docker_build_prepares_quickjs_archive_before_linking( async def fake_ensure_toolchain_image() -> str: return "frameos-cross-test" - async def fake_exec_local_command(_db, _redis, _frame, _cmd, **_kwargs): + async def fake_docker_run(**_kwargs): captured_script["content"] = (temp_dir / "frameos-cross-build.sh").read_text() return 0, "", "" monkeypatch.setattr(compiler, "_ensure_toolchain_image", fake_ensure_toolchain_image) - monkeypatch.setattr("app.utils.cross_compile.exec_local_command", fake_exec_local_command) + compiler.executor = SimpleNamespace(docker_run=fake_docker_run) result = await compiler._run_docker_build(str(build_dir)) script = captured_script["content"] assert result == str(build_dir / "frameos") - assert "Rebuilding QuickJS archive for target" in script assert "make -C quickjs clean" in script assert "make -C quickjs libquickjs.a" in script + assert "Rebuilding QuickJS archive for target" not in script + assert "Indexing QuickJS archive" not in script assert script.index("make -C quickjs libquickjs.a") < script.index("make -j\"$make_jobs\"") + + +@pytest.mark.asyncio +async def test_quickjs_preparation_is_quiet_on_success(tmp_path, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("FRAMEOS_CROSS_CACHE", str(tmp_path / "cross-cache")) + logs: list[tuple[str, str]] = [] + + async def logger(level: str, message: str) -> None: + logs.append((level, message)) + + compiler = CrossCompiler( + db=None, + redis=None, + frame=SimpleNamespace(id=1), + deployer=SimpleNamespace(build_id="build12345678"), + target=TargetMetadata(arch="aarch64", distro="raspios", version="bookworm"), + temp_dir=str(tmp_path / "tmp"), + prebuilt_entry=None, + logger=logger, + ) + prebuilt_dir = tmp_path / "prebuilt" / "quickjs-2026-06-04" + source_dir = tmp_path / "source" + build_dir = tmp_path / "build" + prebuilt_dir.mkdir(parents=True) + source_dir.mkdir() + build_dir.mkdir() + write_component_payload("quickjs", prebuilt_dir, valid=True) + compiler.prebuilt_components["quickjs"] = prebuilt_dir + + await compiler._ensure_quickjs_sources(str(source_dir)) + await compiler._ensure_quickjs_in_build_dir(str(source_dir), build_dir) + + assert logs == [] + assert (source_dir / "quickjs" / "libquickjs.a").exists() + assert (build_dir / "quickjs" / "libquickjs.a").exists() + + +@pytest.mark.asyncio +async def test_modal_toolchain_build_uses_amd64_image_with_target_cross_compiler( + tmp_path, monkeypatch: pytest.MonkeyPatch +): + temp_dir = tmp_path / "tmp" + build_dir = tmp_path / "build" + temp_dir.mkdir() + build_dir.mkdir() + calls: list[tuple[str, str]] = [] + synced_scripts: list[str] = [] + + class FakeModalSandboxSession: + def __init__(self, config, *, logger=None): + self.config = config + self.logger = logger + + async def __aenter__(self): + calls.append(("image", self.config.image)) + calls.append(("resources", f"{self.config.cpu}:{self.config.memory}")) + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def mktemp_dir(self, prefix: str = "frameos-cross-") -> str: + return "/tmp/frameos-cross-test" + + async def sync_dir_tarball(self, local_path: str, remote_path: str) -> None: + calls.append(("sync", f"{local_path}->{remote_path}")) + + async def sync_dir_archive(self, archive_path: str, remote_path: str) -> None: + calls.append(("sync-archive", f"{archive_path}->{remote_path}")) + + async def write_file(self, remote_path: str, content: str, mode: int = 0o644) -> None: + calls.append(("write", remote_path)) + + async def sync_file(self, local_path: str, remote_path: str) -> None: + calls.append(("sync-file", f"{local_path}->{remote_path}")) + if remote_path.endswith("/build.sh"): + synced_scripts.append(Path(local_path).read_text()) + + async def run(self, command: str, **kwargs): + calls.append(("run", command)) + if "find drivers scenes" in command: + return 0, "", "" + return 0, "", "" + + async def download_file(self, remote_path: str, local_path: str) -> None: + calls.append(("download", f"{remote_path}->{local_path}")) + Path(local_path).write_text("binary") + + async def download_dir_tarball(self, remote_path: str, local_path: str) -> None: + calls.append(("download-dir", f"{remote_path}->{local_path}")) + Path(local_path).mkdir(parents=True, exist_ok=True) + (Path(local_path) / "frameos").write_text("binary") + + compiler = CrossCompiler( + db=None, + redis=None, + frame=SimpleNamespace(id=1), + deployer=SimpleNamespace(build_id="build12345678"), + target=TargetMetadata(arch="aarch64", distro="raspios", version="bookworm"), + temp_dir=str(temp_dir), + prebuilt_entry=None, + build_host=ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:latest", + ), + ) + + monkeypatch.setattr("app.utils.build_executor.ModalSandboxSession", FakeModalSandboxSession) + compiler.executor = create_build_executor( + compiler.build_host, + db=None, + redis=None, + frame=compiler.frame, + logger=None, + ) + + result = await compiler._run_docker_build(str(build_dir)) + + assert result == str(build_dir / "frameos") + image_calls = [value for kind, value in calls if kind == "image"] + assert image_calls == ["frameos/frameos-cross-toolchain:debian_bookworm-linux_amd64-latest"] + assert ("resources", "8.0:16384") in calls + run_commands = [value for kind, value in calls if kind == "run"] + assert any( + "bash /tmp/frameos-cross/build.sh" in command + for command in run_commands + ) + assert len(synced_scripts) == 1 + assert "apt-get install -y --no-install-recommends gcc-aarch64-linux-gnu" in synced_scripts[0] + assert "export CC=aarch64-linux-gnu-gcc" in synced_scripts[0] diff --git a/backend/app/tasks/tests/test_frame_deploy_workflow.py b/backend/app/tasks/tests/test_frame_deploy_workflow.py index 3b6bf2952..2c9bc97b3 100644 --- a/backend/app/tasks/tests/test_frame_deploy_workflow.py +++ b/backend/app/tasks/tests/test_frame_deploy_workflow.py @@ -85,8 +85,16 @@ async def exec_command(self, command: str, **kwargs) -> int: return self.setup_exit_code if command.startswith('grep -q "^dtoverlay=vc4-kms-v3d" '): return 1 + if "frameos-firstboot-setup.service" in command or "frameos-setup-reset.sh" in command: + return 1 return 0 + async def run_command(self, command: str, **_kwargs) -> tuple[int, str, str]: + self.commands.append(command) + if "frameos-firstboot-setup.service" in command or "frameos-setup-reset.sh" in command: + return 1, "", "" + return 0, "", "" + async def restart_service(self, service: str) -> None: self.restarted_services.append(service) @@ -274,12 +282,13 @@ async def plan_build(self, **kwargs) -> FrameBinaryPlan: plan = await workflow.plan("full") assert captured_kwargs == [ - { - "allow_cross_compile": True, - "force_cross_compile": True, - "compilation_mode": "static", - } - ] + { + "allow_cross_compile": True, + "force_cross_compile": True, + "allow_on_device_fallback": False, + "compilation_mode": "static", + } + ] assert plan.full_deploy is not None assert plan.full_deploy.target["distro"] == "buildroot" assert plan.full_deploy.package_plans == [] @@ -360,12 +369,13 @@ async def plan_build(self, **kwargs) -> FrameBinaryPlan: assert plan.frame_dict["mode"] == "rpios" assert plan.frame_dict["ssh_user"] == "pi" assert captured_kwargs == [ - { - "allow_cross_compile": False, - "force_cross_compile": False, - "compilation_mode": "precompiled", - } - ] + { + "allow_cross_compile": False, + "force_cross_compile": False, + "allow_on_device_fallback": True, + "compilation_mode": "precompiled", + } + ] assert plan.full_deploy is not None assert plan.full_deploy.target["distro"] == "ubuntu" assert {pkg.name for pkg in plan.full_deploy.package_plans} >= {"hostapd", "imagemagick", "build-essential", "caddy"} @@ -1630,6 +1640,28 @@ def test_legacy_shared_driver_setup_segfault_guard_requires_successful_setup_out ) is False +@pytest.mark.asyncio +async def test_setup_json_reset_cleanup_skips_when_helper_absent(): + frame = SimpleNamespace(id=24, name="NoFirstbootHelper", mode="rpios") + deployer = RecordingDeployer() + deployer.db = None + deployer.redis = None + deployer.frame = frame + workflow = FrameDeployWorkflow( + db=None, + redis=None, + frame=frame, + deployer=deployer, + temp_dir="", + binary_builder=FakeBinaryBuilder(), + ) + + await workflow._remove_setup_json_reset_helper() + + assert not any("Removing setup JSON reset helper" in message for _level, message in deployer.logs) + assert not any("systemctl disable frameos-firstboot-setup.service" in command for command in deployer.commands) + + @pytest.mark.asyncio async def test_remote_build_uses_x86_feature_flags(monkeypatch: pytest.MonkeyPatch, tmp_path): archive_path = tmp_path / "build_build12345678.tar.gz" diff --git a/backend/app/utils/build_environment.py b/backend/app/utils/build_environment.py new file mode 100644 index 000000000..b842d2106 --- /dev/null +++ b/backend/app/utils/build_environment.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Literal + +from sqlalchemy.orm import Session + +from app.models.settings import get_settings_dict + +BuildEnvironmentProvider = Literal["none", "docker", "buildHost", "modal"] +BUILD_ENVIRONMENT_PROVIDERS: set[str] = {"none", "docker", "buildHost", "modal"} + + +def normalize_build_environment_provider(value: object) -> BuildEnvironmentProvider | None: + if value in BUILD_ENVIRONMENT_PROVIDERS: + return value # type: ignore[return-value] + return None + + +def selected_build_environment_provider(settings: dict | None) -> BuildEnvironmentProvider: + settings = settings or {} + raw = settings.get("buildEnvironment") + if isinstance(raw, dict): + provider = normalize_build_environment_provider(raw.get("provider")) + if provider: + return provider + + # Backward-compatible inference for existing installations before the + # provider became a single explicit choice. + modal = settings.get("modalSandbox") + if isinstance(modal, dict) and modal.get("enabled"): + return "modal" + build_host = settings.get("buildHost") + if isinstance(build_host, dict) and build_host.get("enabled"): + return "buildHost" + return "docker" + + +def get_selected_build_environment_provider( + db: Session | None, + project_id: int | None = None, +) -> BuildEnvironmentProvider: + if db is None or project_id is None: + return "docker" + return selected_build_environment_provider(get_settings_dict(db, project_id=project_id)) + + +def server_side_compilation_enabled(settings: dict | None) -> bool: + return selected_build_environment_provider(settings) != "none" diff --git a/backend/app/utils/build_executor.py b/backend/app/utils/build_executor.py new file mode 100644 index 000000000..9b6b016f5 --- /dev/null +++ b/backend/app/utils/build_executor.py @@ -0,0 +1,667 @@ +from __future__ import annotations + +import asyncio +import os +import shlex +import shutil +import tarfile +import tempfile +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath +from typing import Awaitable, Callable, Literal + +from arq import ArqRedis +from sqlalchemy.orm import Session + +from app.models.frame import Frame +from app.models.log import new_log as log +from app.utils.build_environment import BuildEnvironmentProvider +from app.utils.build_host import BuildHostConfig, BuildHostSession +from app.utils.modal_sandbox import ( + ModalSandboxConfig, + ModalSandboxSession, + parse_docker_run_command, + sandbox_sync_paths_for_command, +) + +LogFunc = Callable[[str, str], Awaitable[None]] +RunResult = tuple[int, str | None, str | None] + +MODAL_DEFAULT_CPU = 2.0 +MODAL_DEFAULT_MEMORY = 4096 +MODAL_COMPILE_CPU = 8.0 +MODAL_COMPILE_MEMORY = 16384 +MODAL_COMPOSE_CPU = 4.0 +MODAL_COMPOSE_MEMORY = 8192 + + +@dataclass(slots=True) +class DockerMount: + source: Path + target: str + read_only: bool = False + + +class BuildExecutor: + display_name: str = "local Docker" + uses_local_filesystem: bool = True + uses_container_images_directly: bool = False + connects_on_enter: bool = False + + async def __aenter__(self) -> "BuildExecutor": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: # noqa: ANN001 + return None + + async def run( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + raise NotImplementedError + + async def docker_run( + self, + *, + image: str, + args: list[str], + mounts: list[DockerMount], + env: dict[str, str] | None = None, + workdir: str | None = None, + platform: str | None = None, + ulimits: list[str] | None = None, + workspace: str = "docker-run", + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + command = self._docker_run_command( + image=image, + args=args, + mounts=mounts, + env=env, + workdir=workdir, + platform=platform, + ulimits=ulimits, + ) + return await self.run( + command, + log_command=log_command, + log_output=log_output, + stderr_log_tag=stderr_log_tag, + ) + + async def prepare_docker_build_context(self, dockerfile: Path, workspace: str) -> tuple[str, str]: + return str(dockerfile.parent), str(dockerfile) + + def container_image_reference(self, image: str, resolved_image: str) -> str: + return resolved_image + + def container_platform_for_target(self, target_platform: str | None) -> str | None: + return target_platform + + async def mktemp_dir(self, prefix: str = "frameos-build-") -> str: + raise RuntimeError(f"{self.display_name} does not allocate remote workspaces") + + async def ensure_dir(self, path: str) -> None: + Path(path).mkdir(parents=True, exist_ok=True) + + async def remove_path(self, path: str) -> None: + target = Path(path) + if target.is_dir() and not target.is_symlink(): + shutil.rmtree(target) + else: + target.unlink(missing_ok=True) + + async def sync_dir_tarball(self, local_path: str, remote_path: str) -> None: + local = Path(local_path) + remote = Path(remote_path) + if remote.exists(): + shutil.rmtree(remote) + if local.is_dir(): + shutil.copytree(local, remote, symlinks=True) + + async def sync_file(self, local_path: str, remote_path: str) -> None: + local = Path(local_path) + if not local.is_file(): + return + remote = Path(remote_path) + remote.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(local, remote) + + async def write_file(self, remote_path: str, content: str, mode: int = 0o644) -> None: + path = Path(remote_path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + os.chmod(path, mode) + + async def download_file(self, remote_path: str, local_path: str) -> None: + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(remote_path, local_path) + + async def download_dir_tarball(self, remote_path: str, local_path: str) -> None: + remote = Path(remote_path) + local = Path(local_path) + if not remote.exists(): + return + if local.exists(): + shutil.rmtree(local) + shutil.copytree(remote, local, symlinks=True) + + @staticmethod + def _docker_run_command( + *, + image: str, + args: list[str], + mounts: list[DockerMount], + env: dict[str, str] | None, + workdir: str | None, + platform: str | None, + ulimits: list[str] | None, + ) -> str: + parts = ["docker run --rm"] + if platform: + parts.append(f"--platform {shlex.quote(platform)}") + for ulimit in ulimits or []: + parts.append(f"--ulimit {shlex.quote(ulimit)}") + for mount in mounts: + source = shlex.quote(str(mount.source)) + suffix = ":ro" if mount.read_only else "" + parts.append(f"-v {source}:{shlex.quote(mount.target)}{suffix}") + for key, value in (env or {}).items(): + parts.append(f"-e {key}={shlex.quote(value)}") + if workdir: + parts.append(f"-w {shlex.quote(workdir)}") + parts.append(shlex.quote(image)) + parts.extend(shlex.quote(arg) for arg in args) + return " ".join(parts) + + +class LocalBuildExecutor(BuildExecutor): + def __init__( + self, + *, + db: Session | None, + redis: ArqRedis | None, + frame: Frame, + ) -> None: + self.db = db + self.redis = redis + self.frame = frame + + async def run( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + if log_command: + if self.db and self.redis: + await log( + self.db, + self.redis, + int(self.frame.id), + "stdout", + f"$ {log_command if isinstance(log_command, str) else command}", + ) + else: + print(f"$ {log_command if isinstance(log_command, str) else command}") + + proc = await asyncio.create_subprocess_shell( + command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + async def pump(stream, tag, buf): + pending = "" + + async def _flush(segment: str, *, terminated: bool) -> None: + if not segment: + return + buf.append(f"{segment}\n" if terminated else segment) + if not log_output: + return + if self.db and self.redis: + await log(self.db, self.redis, int(self.frame.id), tag, segment) + else: + print(segment) + + while True: + raw = await stream.read(1024) + if not raw: + break + chunk = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw) + pending += chunk + while True: + newline_index = pending.find("\n") + carriage_index = pending.find("\r") + split_index = -1 + if newline_index != -1 and carriage_index != -1: + split_index = min(newline_index, carriage_index) + elif newline_index != -1: + split_index = newline_index + elif carriage_index != -1: + split_index = carriage_index + if split_index == -1: + break + segment = pending[:split_index].strip("\r") + pending = pending[split_index + 1 :] + await _flush(segment, terminated=True) + pending = pending.strip("\r") + if pending: + await _flush(pending, terminated=False) + + out_buf: list[str] = [] + err_buf: list[str] = [] + await asyncio.gather( + pump(proc.stdout, "stdout", out_buf), + pump(proc.stderr, stderr_log_tag, err_buf), + ) + + exit_code = await proc.wait() + if exit_code and log_output: + if self.db and self.redis: + await log( + self.db, + self.redis, + int(self.frame.id), + "exit_status", + f"The command exited with status {exit_code}", + ) + else: + print(f"The command exited with status {exit_code}") + return exit_code, "".join(out_buf) or None, "".join(err_buf) or None + + +class BuildHostExecutor(BuildExecutor): + uses_local_filesystem = False + connects_on_enter = True + + def __init__( + self, + config: BuildHostConfig, + *, + logger: LogFunc | None = None, + workspace_prefix: str = "frameos-build-", + ) -> None: + self.config = config + self.display_name = f"build host {config.user}@{config.host}:{config.port}" + self.logger = logger + self.workspace_prefix = workspace_prefix + self.session: BuildHostSession | None = None + self.remote_root: PurePosixPath | None = None + + async def __aenter__(self) -> "BuildHostExecutor": + self.session = await BuildHostSession(self.config, logger=self.logger).__aenter__() + self.remote_root = PurePosixPath(await self.session.mktemp_dir(self.workspace_prefix)) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: # noqa: ANN001 + if self.session: + await self.session.__aexit__(exc_type, exc, tb) + self.session = None + self.remote_root = None + + def workspace_path(self, name: str) -> PurePosixPath: + if self.remote_root is None: + raise RuntimeError("Build host workspace is not available") + return self.remote_root / name + + async def run( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + return await self.session.run(command, log_command=log_command, log_output=log_output) + + async def docker_run( + self, + *, + image: str, + args: list[str], + mounts: list[DockerMount], + env: dict[str, str] | None = None, + workdir: str | None = None, + platform: str | None = None, + ulimits: list[str] | None = None, + workspace: str = "docker-run", + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + remote_mounts: list[DockerMount] = [] + workspace_root = self.workspace_path(workspace) + await self.session.remove_path(str(workspace_root)) + for index, mount in enumerate(mounts): + remote_source = workspace_root / f"mount-{index}-{mount.source.name}" + if mount.source.is_dir(): + await self.session.sync_dir_tarball(str(mount.source), str(remote_source)) + elif mount.source.is_file(): + await self.session.sync_file(str(mount.source), str(remote_source)) + remote_mounts.append(DockerMount(Path(str(remote_source)), mount.target, mount.read_only)) + + command = self._docker_run_command( + image=image, + args=args, + mounts=remote_mounts, + env=env, + workdir=workdir, + platform=platform, + ulimits=ulimits, + ) + status, out, err = await self.run( + command, + log_command=log_command, + log_output=log_output, + stderr_log_tag=stderr_log_tag, + ) + if status == 0: + for mount, remote_mount in zip(mounts, remote_mounts): + if mount.read_only: + continue + if mount.source.is_dir(): + await self.session.download_dir_tarball(str(remote_mount.source), str(mount.source)) + elif mount.source.is_file(): + await self.session.download_file(str(remote_mount.source), str(mount.source)) + return status, out, err + + async def prepare_docker_build_context(self, dockerfile: Path, workspace: str) -> tuple[str, str]: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + remote_dir = self.workspace_path(workspace) + dockerfile_path = remote_dir / dockerfile.name + await self.session.ensure_dir(str(remote_dir)) + await self.session.write_file(str(dockerfile_path), dockerfile.read_text(encoding="utf-8")) + return str(remote_dir), str(dockerfile_path) + + async def mktemp_dir(self, prefix: str = "frameos-build-") -> str: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + return await self.session.mktemp_dir(prefix) + + async def ensure_dir(self, path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.ensure_dir(path) + + async def remove_path(self, path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.remove_path(path) + + async def sync_dir_tarball(self, local_path: str, remote_path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.sync_dir_tarball(local_path, remote_path) + + async def sync_file(self, local_path: str, remote_path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.sync_file(local_path, remote_path) + + async def write_file(self, remote_path: str, content: str, mode: int = 0o644) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.write_file(remote_path, content, mode=mode) + + async def download_file(self, remote_path: str, local_path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.download_file(remote_path, local_path) + + async def download_dir_tarball(self, remote_path: str, local_path: str) -> None: + if self.session is None: + raise RuntimeError("Build host executor is not connected") + await self.session.download_dir_tarball(remote_path, local_path) + + +class ModalBuildExecutor(BuildExecutor): + uses_local_filesystem = False + uses_container_images_directly = True + + def __init__( + self, + config: ModalSandboxConfig, + *, + logger: LogFunc | None = None, + ) -> None: + self.config = config + self.logger = logger + self.display_name = f"Modal sandbox app {config.app_name} ({config.image})" + + def container_image_reference(self, image: str, resolved_image: str) -> str: + # Modal wraps registry images in its own Image build. Plain tags are + # accepted more reliably than tag+digest references there; Docker and + # build-host executors keep using digest-pinned references. + return image + + def container_platform_for_target(self, target_platform: str | None) -> str | None: + # Modal registry images must be linux/amd64. Non-amd64 targets need an + # amd64 image that contains the target cross compiler. + if self._requires_amd64_container(target_platform): + return "linux/amd64" + return target_platform + + @staticmethod + def _requires_amd64_container(platform: str | None) -> bool: + if not platform: + return False + normalized = platform.lower() + return normalized not in {"linux/amd64", "amd64", "x86_64"} + + async def _sandbox_log(self, level: str, message: str) -> None: + if self.logger: + await self.logger(level, message) + + def _sandbox_logger(self, stderr_log_tag: Literal["stderr", "stdout"]) -> LogFunc: + async def sandbox_log(level: str, message: str) -> None: + tag = "stdout" if level == "stderr" and stderr_log_tag == "stdout" else level + if self.logger: + await self.logger(tag, message) + + return sandbox_log + + def _config_with_resources( + self, + *, + image: str | None = None, + workspace: str = "docker-run", + enable_docker: bool = False, + ) -> ModalSandboxConfig: + cpu, memory = self._resource_profile(workspace) + return replace( + self.config, + image=image if image is not None else self.config.image, + enable_docker=enable_docker, + cpu=self.config.cpu if self.config.cpu is not None else cpu, + memory=self.config.memory if self.config.memory is not None else memory, + ) + + @staticmethod + def _resource_profile(workspace: str) -> tuple[float, int]: + if workspace in {"cross-compile", "buildroot-image"}: + return MODAL_COMPILE_CPU, MODAL_COMPILE_MEMORY + if workspace in {"compose", "boot-patch"}: + return MODAL_COMPOSE_CPU, MODAL_COMPOSE_MEMORY + return MODAL_DEFAULT_CPU, MODAL_DEFAULT_MEMORY + + async def run( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + docker_run = parse_docker_run_command(command) + if docker_run: + return await self.docker_run( + image=docker_run.image, + args=docker_run.args or [], + mounts=[ + DockerMount(mount.source, mount.target, read_only=mount.read_only) + for mount in docker_run.mounts + ], + env=docker_run.env, + workdir=docker_run.workdir, + log_command=log_command, + log_output=log_output, + stderr_log_tag=stderr_log_tag, + ) + + sync_paths = sandbox_sync_paths_for_command(command) + config = self._config_with_resources(workspace="command") + async with ModalSandboxSession(config, logger=self._sandbox_logger(stderr_log_tag)) as sandbox: + if sync_paths: + await self._sandbox_logger(stderr_log_tag)( + "stdout", + f"Preparing Modal sandbox with {len(sync_paths)} local path" + + ("s" if len(sync_paths) != 1 else ""), + ) + for path in sync_paths: + if path.is_dir(): + await sandbox.sync_dir_tarball(str(path), str(path)) + elif path.is_file(): + await sandbox.sync_file(str(path), str(path)) + + status, out, err = await sandbox.run(command, log_command=log_command, log_output=log_output) + + if status == 0: + for path in sync_paths: + if path.is_dir(): + await sandbox.download_dir_tarball(str(path), str(path)) + elif path.is_file(): + await sandbox.download_file(str(path), str(path)) + return status, out, err + + async def docker_run( + self, + *, + image: str, + args: list[str], + mounts: list[DockerMount], + env: dict[str, str] | None = None, + workdir: str | None = None, + platform: str | None = None, + ulimits: list[str] | None = None, + workspace: str = "docker-run", + log_command: str | bool = True, + log_output: bool = True, + stderr_log_tag: Literal["stderr", "stdout"] = "stderr", + ) -> RunResult: + if self._requires_amd64_container(platform): + message = ( + "Modal sandboxes can only start linux/amd64 registry images directly. " + "Use an amd64 toolchain image with a target cross compiler instead of " + f"requesting {platform} for {image}." + ) + await self._sandbox_logger(stderr_log_tag)("stderr", message) + return 125, None, f"{message}\n" + + config = self._config_with_resources(image=image, workspace=workspace, enable_docker=False) + run_command = " ".join(shlex.quote(arg) for arg in args) if args else "true" + if workdir: + run_command = f"cd {shlex.quote(workdir)} && {run_command}" + for ulimit in ulimits or []: + if ulimit.startswith("nofile="): + soft = ulimit.removeprefix("nofile=").split(":", 1)[0] + run_command = f"ulimit -n {shlex.quote(soft)} && {run_command}" + if env: + exports = " ".join(f"{key}={shlex.quote(value)}" for key, value in env.items()) + run_command = f"export {exports}; {run_command}" + + prepared_archives: dict[Path, Path] = {} + try: + for mount in mounts: + if not mount.source.is_dir(): + continue + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp: + archive_path = Path(tmp.name) + with tarfile.open(archive_path, "w:gz") as tar: + tar.add(mount.source, arcname=".") + prepared_archives[mount.source] = archive_path + + async with ModalSandboxSession(config, logger=self._sandbox_logger(stderr_log_tag)) as sandbox: + if mounts: + await self._sandbox_log( + "stdout", + f"Preparing Modal sandbox from {image} with {len(mounts)} mount" + + ("s" if len(mounts) != 1 else ""), + ) + for mount in mounts: + if mount.source.is_dir(): + await sandbox.sync_dir_archive(str(prepared_archives[mount.source]), mount.target) + elif mount.source.is_file(): + await sandbox.sync_file(str(mount.source), mount.target) + + status, out, err = await sandbox.run( + run_command, + log_command=log_command if log_command is not True else f"Modal run {image}", + log_output=log_output, + ) + if status == 0: + for mount in mounts: + if mount.read_only: + continue + if mount.source.is_dir(): + await sandbox.download_dir_tarball(mount.target, str(mount.source)) + elif mount.source.is_file(): + await sandbox.download_file(mount.target, str(mount.source)) + finally: + for archive_path in prepared_archives.values(): + archive_path.unlink(missing_ok=True) + return status, out, err + + +def build_executor_display_name(config: BuildHostConfig | ModalSandboxConfig) -> str: + if isinstance(config, ModalSandboxConfig): + return f"Modal sandbox app {config.app_name} ({config.image})" + return f"build host {config.user}@{config.host}:{config.port}" + + +def build_executor_kind_name(config: BuildHostConfig | ModalSandboxConfig) -> str: + if isinstance(config, ModalSandboxConfig): + return "Modal sandbox" + return "build host" + + +def build_environment_requires_executor_config(provider: BuildEnvironmentProvider) -> bool: + return provider in {"buildHost", "modal"} + + +def ensure_build_executor_configured( + provider: BuildEnvironmentProvider, + config: BuildHostConfig | ModalSandboxConfig | None, +) -> None: + if build_environment_requires_executor_config(provider) and config is None: + raise RuntimeError(f"Selected build environment '{provider}' is not configured") + + +def create_build_executor( + config: BuildHostConfig | ModalSandboxConfig | None, + *, + db: Session | None, + redis: ArqRedis | None, + frame: Frame, + logger: LogFunc | None = None, + workspace_prefix: str = "frameos-build-", +) -> BuildExecutor: + if isinstance(config, ModalSandboxConfig): + return ModalBuildExecutor(config, logger=logger) + if isinstance(config, BuildHostConfig): + return BuildHostExecutor(config, logger=logger, workspace_prefix=workspace_prefix) + return LocalBuildExecutor(db=db, redis=redis, frame=frame) diff --git a/backend/app/utils/build_host.py b/backend/app/utils/build_host.py index 77aac485d..43de8d298 100644 --- a/backend/app/utils/build_host.py +++ b/backend/app/utils/build_host.py @@ -3,6 +3,7 @@ import asyncio import os import shlex +import shutil import tarfile import tempfile from dataclasses import dataclass @@ -13,6 +14,8 @@ from sqlalchemy.orm import Session from app.models.settings import get_settings_dict +from app.utils.build_environment import selected_build_environment_provider +from app.utils.modal_sandbox import ModalSandboxConfig, get_modal_sandbox_config LogFunc = Callable[[str, str], Awaitable[None]] @@ -45,9 +48,17 @@ def get_build_host_config(db: Session | None, project_id: int | None = None) -> if db is None: return None settings = get_settings_dict(db, project_id=project_id) + if selected_build_environment_provider(settings) != "buildHost": + return None return BuildHostConfig.from_settings(settings.get("buildHost")) +def get_build_executor_config(db: Session | None, project_id: int | None = None) -> BuildHostConfig | ModalSandboxConfig | None: + if db is None or project_id is None: + return None + return get_modal_sandbox_config(db, project_id) or get_build_host_config(db, project_id) + + class BuildHostSession: def __init__( self, @@ -206,6 +217,14 @@ async def sync_dir_tarball(self, local_path: str, remote_path: str) -> None: finally: archive_path.unlink(missing_ok=True) + async def sync_file(self, local_path: str, remote_path: str) -> None: + if not self._conn: + raise RuntimeError("Build host session is not connected") + if not Path(local_path).is_file(): + return + await self.ensure_dir(str(Path(remote_path).parent)) + await asyncssh.scp(local_path, (self._conn, remote_path), preserve=True) + async def write_file(self, remote_path: str, content: str, mode: int = 0o644) -> None: if not self._conn: raise RuntimeError("Build host session is not connected") @@ -224,3 +243,44 @@ async def download_file(self, remote_path: str, local_path: str) -> None: raise RuntimeError("Build host session is not connected") Path(local_path).parent.mkdir(parents=True, exist_ok=True) await asyncssh.scp((self._conn, remote_path), local_path) + + async def download_dir_tarball(self, remote_path: str, local_path: str) -> None: + if not self._conn: + raise RuntimeError("Build host session is not connected") + + fd, tmp_path = tempfile.mkstemp(suffix=".tar.gz") + os.close(fd) + archive_path = Path(tmp_path) + remote_archive = f"{remote_path}.download.tar.gz" + local = Path(local_path) + try: + status, _out, _err = await self.run( + " ".join( + [ + "test -e", + shlex.quote(remote_path), + "&& tar -czf", + shlex.quote(remote_archive), + "-C", + shlex.quote(remote_path), + ".", + ] + ), + log_command=False, + log_output=False, + ) + if status != 0: + return + await self.download_file(remote_archive, str(archive_path)) + if local.exists() and local.is_dir(): + for child in local.iterdir(): + if child.is_dir(): + shutil.rmtree(child) + else: + child.unlink() + local.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive_path, "r:gz") as tar: + tar.extractall(local) + finally: + archive_path.unlink(missing_ok=True) + await self.remove_path(remote_archive) diff --git a/backend/app/utils/cross_compile.py b/backend/app/utils/cross_compile.py index ef6d08b41..aeea4cdae 100644 --- a/backend/app/utils/cross_compile.py +++ b/backend/app/utils/cross_compile.py @@ -27,8 +27,20 @@ fetch_prebuilt_manifest, resolve_prebuilt_target, ) -from app.utils.build_host import BuildHostConfig, BuildHostSession -from app.utils.local_exec import exec_local_command +from app.utils.build_host import BuildHostConfig +from app.utils.build_executor import ( + BuildExecutor, + DockerMount, + build_executor_display_name, + create_build_executor, +) +from app.utils.cross_toolchain_packages import ( + TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS, + TARGET_CROSS_TOOLCHAIN_PACKAGES, + TARGET_CROSS_TOOLCHAINS, + TargetCrossToolchain, +) +from app.utils.modal_sandbox import ModalSandboxConfig icon = "🔶" @@ -87,7 +99,6 @@ class TargetMetadata: "armhf": "linux/arm/v7", "armv6l": "linux/arm/v6", } - def can_cross_compile_target(arch: str | None) -> bool: """Return ``True`` when *arch* has a known Docker platform mapping.""" @@ -168,7 +179,7 @@ def __init__( prebuilt_target: str | None = None, logger: LogFunc | None = None, build_dir: str | Path | None = None, - build_host: BuildHostConfig | None = None, + build_host: BuildHostConfig | ModalSandboxConfig | None = None, output_name: str = "frameos", compile_script_name: str = "compile_frameos.sh", needs_quickjs: bool = True, @@ -195,8 +206,7 @@ def __init__( self.prebuilt_timeout = PREBUILT_TIMEOUT self.logger = logger self.build_host = build_host - self._build_host_session: BuildHostSession | None = None - self._remote_root: Path | None = None + self.executor: BuildExecutor | None = None self.output_name = output_name self.compile_script_name = compile_script_name self.needs_quickjs = needs_quickjs @@ -206,24 +216,35 @@ def __init__( (self.sysroot_dir / rel).mkdir(parents=True, exist_ok=True) async def build(self, source_dir: str) -> str: + executor = create_build_executor( + self.build_host, + db=self.db, + redis=self.redis, + frame=self.frame, + logger=self._log, + workspace_prefix="frameos-cross-", + ) if self.build_host: + connection_action = ( + f"Connecting to {build_executor_display_name(self.build_host)}" + if executor.connects_on_enter + else f"Using {build_executor_display_name(self.build_host)}; sandbox will be created when the build command starts" + ) await self._log( "stdout", - f"{icon} Connecting to build host {self.build_host.user}@{self.build_host.host}:{self.build_host.port}", + f"{icon} {connection_action}", ) - async with BuildHostSession(self.build_host, logger=self._log) as session: - self._build_host_session = session - self._remote_root = Path(await session.mktemp_dir("frameos-cross-")) + async with executor: + self.executor = executor + if self.build_host and executor.connects_on_enter: await self._log( "stdout", - f"🟢 Connected to build host {self.build_host.user}@{self.build_host.host}:{self.build_host.port} for cross compilation", + f"Connected to {build_executor_display_name(self.build_host)} for cross compilation", ) - try: - return await self._build_with_context(source_dir) - finally: - self._build_host_session = None - self._remote_root = None - return await self._build_with_context(source_dir) + try: + return await self._build_with_context(source_dir) + finally: + self.executor = None async def _build_with_context(self, source_dir: str) -> str: await self._log( @@ -259,14 +280,18 @@ async def _build_with_context(self, source_dir: str) -> str: return binary_path async def _prepare_sysroot(self) -> None: - await self._log( - "stdout", - f"{icon} Using default include/lib paths without remote sysroot synchronization", - ) + return async def _run_docker_build(self, build_dir: str) -> str: build_dir = os.path.abspath(build_dir) script_path = self.temp_dir / "frameos-cross-build.sh" + target_platform = self._platform() + container_platform = self._container_platform() + target_cross_toolchain = self._target_cross_toolchain(container_platform) + if container_platform != target_platform and target_cross_toolchain is None: + raise RuntimeError( + f"No target cross compiler bootstrap is configured for {target_platform} on {container_platform}" + ) include_candidates = ( [f"/sysroot{path}" for path in sorted(self._sysroot_include_dirs)] if self._sysroot_include_dirs @@ -290,12 +315,28 @@ async def _run_docker_build(self, build_dir: str) -> str: f"{icon} Enabling CPU feature flags for cross-compile: " + " ".join(feature_flags), ) + if target_cross_toolchain: + await self._log( + "stdout", + f"{icon} Using {container_platform} toolchain image with " + f"{target_cross_toolchain.triplet} compiler for {target_platform}", + ) + include_dirs = self._dedupe_preserve_order( + [f"/usr/include/{target_cross_toolchain.triplet}", *include_dirs] + ) + lib_dirs = self._dedupe_preserve_order( + [f"/usr/lib/{target_cross_toolchain.triplet}", *lib_dirs] + ) include_flags = [f"-I{path}" for path in include_dirs] extra_cflags_parts = [*feature_flags, *include_flags] extra_cflags = ( shlex.quote(" ".join(extra_cflags_parts)) if extra_cflags_parts else "''" ) extra_libs = shlex.quote(" ".join(f"-L{path}" for path in lib_dirs)) if lib_dirs else "''" + target_toolchain_script = indent( + self._target_cross_toolchain_setup_script(target_cross_toolchain), + " " * 16, + ) prepare_quickjs_script = indent(self._prepare_quickjs_archive_script(), " " * 16) make_jobs = (os.environ.get("FRAMEOS_CROSS_MAKE_JOBS") or "").strip() make_jobs_assignment = ( @@ -316,6 +357,8 @@ async def _run_docker_build(self, build_dir: str) -> str: log_debug "Container uname: $(uname -a)" log_debug "Working directory before build: $(pwd)" + {target_toolchain_script} + extra_cflags={extra_cflags} extra_libs={extra_libs} {make_jobs_assignment} @@ -341,98 +384,28 @@ async def _run_docker_build(self, build_dir: str) -> str: image = await self._ensure_toolchain_image() - if self._build_host_session: - return await self._run_remote_docker_build(build_dir, script_content, image) - script_path.write_text(script_content) os.chmod(script_path, 0o755) - - docker_cmd = " ".join( - [ - "docker run --rm", - f"--platform {self._platform()}", - f"-v {shlex.quote(build_dir)}:/src", - f"-v {shlex.quote(str(self.sysroot_dir))}:/sysroot:ro", - f"-v {shlex.quote(str(script_path))}:/tmp/frameos-cross/build.sh:ro", - "-w /src", - shlex.quote(image), - "bash /tmp/frameos-cross/build.sh", - ] - ) - - status, _, err = await exec_local_command( - self.db, - self.redis, - self.frame, - docker_cmd, + if self.executor is None: + raise RuntimeError("Build executor unavailable during cross compilation") + + status, _, err = await self.executor.docker_run( + image=image, + platform=container_platform, + mounts=[ + DockerMount(Path(build_dir), "/src"), + DockerMount(self.sysroot_dir, "/sysroot", read_only=True), + DockerMount(script_path, "/tmp/frameos-cross/build.sh", read_only=True), + ], + workdir="/src", + args=["bash", "/tmp/frameos-cross/build.sh"], + workspace="cross-compile", log_command="docker run (cross compile)", ) if status != 0: raise RuntimeError(f"Cross compilation failed: {err or 'see logs'}") return os.path.join(build_dir, self.output_name) - async def _run_remote_docker_build( - self, build_dir: str, script_content: str, image: str - ) -> str: - if not self._build_host_session or not self._remote_root: - raise RuntimeError("Build host session unavailable during cross compilation") - - host = self._build_host_session - remote_build_dir = str(self._remote_root / "src") - remote_sysroot_dir = str(self._remote_root / "sysroot") - remote_script_path = str(self._remote_root / "build.sh") - - build_dir_size = self._dir_size_bytes(Path(build_dir)) - await self._log( - "stdout", - f"{icon} Syncing build directory ({self._format_size(build_dir_size)}) to build host" - ) - await host.sync_dir_tarball(build_dir, remote_build_dir) - sysroot_size = self._dir_size_bytes(self.sysroot_dir) - await self._log( - "stdout", - f"{icon} Syncing sysroot ({self._format_size(sysroot_size)}) to build host" - ) - await host.sync_dir_tarball(str(self.sysroot_dir), remote_sysroot_dir) - await host.write_file(remote_script_path, script_content, mode=0o755) - - docker_cmd = " ".join( - [ - "docker run --rm", - f"--platform {self._platform()}", - f"-v {shlex.quote(remote_build_dir)}:/src", - f"-v {shlex.quote(remote_sysroot_dir)}:/sysroot:ro", - f"-v {shlex.quote(remote_script_path)}:/tmp/build.sh:ro", - "-w /src", - shlex.quote(image), - "bash /tmp/build.sh", - ] - ) - - status, _, err = await host.run( - docker_cmd, - log_command="docker run (build host cross compile)", - ) - if status != 0: - raise RuntimeError(f"Cross compilation failed: {err or 'see logs'}") - - local_binary = os.path.join(build_dir, self.output_name) - await host.download_file(f"{remote_build_dir}/{self.output_name}", local_binary) - status, stdout, _err = await host.run( - f"cd {shlex.quote(remote_build_dir)} && find drivers scenes -type f -name '*.so' 2>/dev/null || true", - log_command=False, - log_output=False, - ) - if status == 0 and stdout: - for rel_path in stdout.splitlines(): - rel_path = rel_path.strip() - if not rel_path: - continue - local_path = os.path.join(build_dir, rel_path) - os.makedirs(os.path.dirname(local_path), exist_ok=True) - await host.download_file(f"{remote_build_dir}/{rel_path}", local_path) - return local_binary - async def _run_command( self, command: str, @@ -440,14 +413,9 @@ async def _run_command( log_command: str | bool = True, log_output: bool = True, ) -> tuple[int, str | None, str | None]: - if self._build_host_session: - return await self._build_host_session.run( - command, log_command=log_command, log_output=log_output - ) - return await exec_local_command( - self.db, - self.redis, - self.frame, + if self.executor is None: + raise RuntimeError("Build executor unavailable during cross compilation") + return await self.executor.run( command, log_command=log_command, log_output=log_output, @@ -463,10 +431,6 @@ def _cpu_feature_cflags(self) -> list[str]: async def _ensure_quickjs_sources(self, source_dir: str) -> None: quickjs_root = Path(source_dir) / "quickjs" - await self._log( - "stdout", - f"{icon} Ensuring QuickJS sources are available at {quickjs_root} (exists={quickjs_root.exists()})", - ) await self._ensure_quickjs_tree( quickjs_root, context="source directory", @@ -490,10 +454,6 @@ async def _generate_c_sources(self, source_dir: str) -> Path: async def _ensure_quickjs_in_build_dir(self, source_dir: str, build_dir: Path) -> None: dest = Path(build_dir) / "quickjs" source_quickjs = Path(source_dir) / "quickjs" - await self._log( - "stdout", - f"{icon} Ensuring QuickJS assets exist within build dir {dest} (exists={dest.exists()})", - ) fallback_src = source_quickjs if source_quickjs.exists() else None await self._ensure_quickjs_tree( dest, @@ -514,27 +474,42 @@ async def _ensure_quickjs_tree( ) -> None: libquickjs = dest / "libquickjs.a" prebuilt_quickjs = self.prebuilt_components.get("quickjs") - if prebuilt_quickjs: - await self._log( - "stdout", - f"{icon} Staging prebuilt QuickJS component from {prebuilt_quickjs} into {dest}", - ) - self._stage_prebuilt_quickjs(dest) + try: + if prebuilt_quickjs: + self._stage_prebuilt_quickjs(dest) - if not libquickjs.exists() and fallback_src: - if dest.exists(): - shutil.rmtree(dest) + if not libquickjs.exists() and fallback_src: + if dest.exists(): + shutil.rmtree(dest) + shutil.copytree(fallback_src, dest, dirs_exist_ok=True) + except Exception as exc: await self._log( - "stdout", - f"{icon} Copying QuickJS tree from {fallback_src} into {dest}", + "stderr", + f"{icon} Failed to prepare QuickJS artifacts for {context}: {exc}", + ) + await self._log_quickjs_probe( + dest.parent, + context, + expected=libquickjs, + prebuilt=prebuilt_quickjs, + fallback=fallback_src, ) - shutil.copytree(fallback_src, dest, dirs_exist_ok=True) + raise if libquickjs.exists(): - await self._log("stdout", f"{icon} Found QuickJS archive at {libquickjs}") return - await self._log_quickjs_probe(dest.parent, context) + await self._log( + "stderr", + f"{icon} QuickJS artifacts missing for {context}: expected {libquickjs}", + ) + await self._log_quickjs_probe( + dest.parent, + context, + expected=libquickjs, + prebuilt=prebuilt_quickjs, + fallback=fallback_src, + ) raise RuntimeError(error_message) async def _prepare_prebuilt_components(self) -> None: @@ -561,11 +536,8 @@ def _prepare_quickjs_archive_script(self) -> str: """ if [ -d quickjs ]; then if [ -f quickjs/Makefile ]; then - log_debug "Rebuilding QuickJS archive for target" make -C quickjs clean >/dev/null make -C quickjs libquickjs.a - elif [ -f quickjs/libquickjs.a ]; then - log_debug "Indexing QuickJS archive" fi if [ -f quickjs/libquickjs.a ]; then ranlib quickjs/libquickjs.a @@ -807,11 +779,25 @@ def _format_size(size: int) -> str: return f"{value:.1f} TiB" - async def _log_quickjs_probe(self, root: Path, context: str) -> None: + async def _log_quickjs_probe( + self, + root: Path, + context: str, + *, + expected: Path | None = None, + prebuilt: Path | None = None, + fallback: Path | None = None, + ) -> None: await self._log( "stderr", f"{icon} Probing {context} {root} for QuickJS artifacts", ) + if expected: + await self._log("stderr", f" - Expected archive: {expected}") + if prebuilt: + await self._log("stderr", f" - Prebuilt source: {prebuilt}") + if fallback: + await self._log("stderr", f" - Fallback source: {fallback}") libs = sorted(root.rglob("libquickjs.a")) headers = sorted(root.rglob("quickjs.h")) folders = [p for p in root.rglob("quickjs") if p.is_dir()] @@ -842,6 +828,56 @@ def _platform(self) -> str: return str(self.target.platform) return PLATFORM_MAP.get(self.target.arch, "linux/amd64") + def _container_platform(self) -> str: + platform = self._platform() + if self.executor is None: + return platform + platform_for_target = getattr(self.executor, "container_platform_for_target", None) + if platform_for_target is None: + return platform + return platform_for_target(platform) or platform + + def _target_cross_toolchain(self, container_platform: str | None = None) -> TargetCrossToolchain | None: + target_platform = self._platform() + if (container_platform or target_platform) == target_platform: + return None + return TARGET_CROSS_TOOLCHAINS.get(target_platform) + + def _target_cross_toolchain_setup_script(self, toolchain: TargetCrossToolchain | None) -> str: + if toolchain is None: + return "" + package_list = " ".join(shlex.quote(package) for package in toolchain.packages) + pkg_config_libdir = ( + f"/usr/lib/{toolchain.triplet}/pkgconfig:" + f"/usr/share/pkgconfig" + ) + return dedent( + f""" + if ! command -v {shlex.quote(toolchain.cc)} >/dev/null 2>&1 || ! test -e /usr/lib/{shlex.quote(toolchain.triplet)}/libssl.so; then + log_debug "Installing {toolchain.triplet} cross compiler and target libraries" + if ! command -v apt-get >/dev/null 2>&1; then + log_debug "apt-get is required to install target cross compiler packages" + exit 1 + fi + export DEBIAN_FRONTEND=noninteractive + if command -v dpkg >/dev/null 2>&1 && ! dpkg --print-foreign-architectures | grep -qx {shlex.quote(toolchain.dpkg_arch)}; then + dpkg --add-architecture {shlex.quote(toolchain.dpkg_arch)} + fi + apt-get update + apt-get install -y --no-install-recommends {package_list} + rm -rf /var/lib/apt/lists/* + fi + export CC={shlex.quote(toolchain.cc)} + export PKG_CONFIG_LIBDIR={shlex.quote(pkg_config_libdir)} + log_debug "Using target compiler: $CC" + """ + ).strip() + + def _target_cross_toolchain_build_args(self, container_platform: str) -> tuple[str, str]: + if container_platform != "linux/amd64": + return "", "" + return " ".join(TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS), " ".join(TARGET_CROSS_TOOLCHAIN_PACKAGES) + def _docker_image(self) -> str: if getattr(self.target, "image", None): return str(self.target.image) @@ -861,9 +897,9 @@ def _docker_image(self) -> str: safe_version = version or default_version return f"{base}:{safe_version}" - def _toolchain_image(self) -> str: + def _toolchain_image(self, platform_override: str | None = None) -> str: base = self._sanitize(self._docker_image().replace("/", "_")) - platform = self._sanitize(self._platform().replace("/", "_")) + platform = self._sanitize((platform_override or self._platform()).replace("/", "_")) slug = f"{base}-{platform}" if CROSS_TOOLCHAIN_IMAGE: try: @@ -878,8 +914,8 @@ def _toolchain_image(self) -> str: tag = f"{slug}-{TOOLCHAIN_IMAGE_TAG}" if TOOLCHAIN_IMAGE_TAG else slug return f"{TOOLCHAIN_IMAGE_REPO}:{tag}" - def _resolved_toolchain_image(self) -> str: - image = self._toolchain_image() + def _resolved_toolchain_image(self, platform_override: str | None = None) -> str: + image = self._toolchain_image(platform_override) if CROSS_TOOLCHAIN_IMAGE: return image digest = _toolchain_digest_map().get(image) @@ -887,14 +923,19 @@ def _resolved_toolchain_image(self) -> str: return f"{image}@{digest}" return image - def _legacy_toolchain_image(self) -> str: + def _legacy_toolchain_image(self, platform_override: str | None = None) -> str: base = self._sanitize(self._docker_image().replace("/", "_")) - platform = self._sanitize(self._platform().replace("/", "_")) + platform = self._sanitize((platform_override or self._platform()).replace("/", "_")) return f"frameos-cross-{base}-{platform}-v1" async def _ensure_toolchain_image(self) -> str: - image = self._toolchain_image() - resolved_image = self._resolved_toolchain_image() + container_platform = self._container_platform() + image = self._toolchain_image(container_platform) + resolved_image = self._resolved_toolchain_image(container_platform) + if self.executor is None: + raise RuntimeError("Build executor unavailable during cross compilation") + if self.executor.uses_container_images_directly: + return self.executor.container_image_reference(image, resolved_image) if not TOOLCHAIN_FORCE_LOCAL_BUILD: status, _out, _err = await self._run_command( f"docker image inspect {shlex.quote(resolved_image)} >/dev/null 2>&1", @@ -913,7 +954,7 @@ async def _ensure_toolchain_image(self) -> str: if status == 0: return image - legacy_image = self._legacy_toolchain_image() + legacy_image = self._legacy_toolchain_image(container_platform) status, _out, _err = await self._run_command( f"docker image inspect {shlex.quote(legacy_image)} >/dev/null 2>&1", log_command=False, @@ -952,27 +993,20 @@ async def _ensure_toolchain_image(self) -> str: "Cross toolchain Dockerfile is missing; expected at backend/tools/cross-toolchain.Dockerfile", ) - if self._build_host_session: - if not self._remote_root: - raise RuntimeError("Build host workspace missing for toolchain build") - remote_dir = self._remote_root / "cross-toolchain" - dockerfile_path = remote_dir / Path(dockerfile).name - await self._build_host_session.ensure_dir(str(remote_dir)) - await self._build_host_session.write_file( - str(dockerfile_path), Path(dockerfile).read_text() - ) - context_dir = str(remote_dir) - dockerfile_arg = str(dockerfile_path) - else: - context_dir = str(Path(dockerfile).parent) - dockerfile_arg = dockerfile + context_dir, dockerfile_arg = await self.executor.prepare_docker_build_context( + Path(dockerfile), + "cross-toolchain", + ) + target_cross_dpkg_archs, target_cross_packages = self._target_cross_toolchain_build_args(container_platform) build_cmd = " ".join( [ "docker buildx build --load", - f"--platform {self._platform()}", + f"--platform {container_platform}", f"--build-arg BASE_IMAGE={shlex.quote(self._docker_image())}", f"--build-arg TOOLCHAIN_PACKAGES={shlex.quote(TOOLCHAIN_PACKAGES)}", + f"--build-arg TARGET_CROSS_DPKG_ARCHS={shlex.quote(target_cross_dpkg_archs)}", + f"--build-arg TARGET_CROSS_PACKAGES={shlex.quote(target_cross_packages)}", f"-t {shlex.quote(image)}", f"-f {shlex.quote(dockerfile_arg)}", shlex.quote(context_dir), @@ -1029,7 +1063,7 @@ async def build_binary_with_cross_toolchain( prebuilt_target: str | None = None, target_override: TargetMetadata | None = None, logger: LogFunc | None = None, - build_host: BuildHostConfig | None = None, + build_host: BuildHostConfig | ModalSandboxConfig | None = None, ) -> str: arch: str | None distro: str | None diff --git a/backend/app/utils/cross_toolchain_packages.py b/backend/app/utils/cross_toolchain_packages.py new file mode 100644 index 000000000..abf28e5df --- /dev/null +++ b/backend/app/utils/cross_toolchain_packages.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class TargetCrossToolchain: + dpkg_arch: str + triplet: str + cc: str + packages: tuple[str, ...] + + +TARGET_CROSS_TOOLCHAINS = { + "linux/arm64": TargetCrossToolchain( + dpkg_arch="arm64", + triplet="aarch64-linux-gnu", + cc="aarch64-linux-gnu-gcc", + packages=( + "gcc-aarch64-linux-gnu", + "g++-aarch64-linux-gnu", + "pkg-config", + "zlib1g-dev:arm64", + "libssl-dev:arm64", + "libffi-dev:arm64", + "libjpeg-dev:arm64", + "libfreetype6-dev:arm64", + "libevdev-dev:arm64", + ), + ), + "linux/arm/v7": TargetCrossToolchain( + dpkg_arch="armhf", + triplet="arm-linux-gnueabihf", + cc="arm-linux-gnueabihf-gcc", + packages=( + "gcc-arm-linux-gnueabihf", + "g++-arm-linux-gnueabihf", + "pkg-config", + "zlib1g-dev:armhf", + "libssl-dev:armhf", + "libffi-dev:armhf", + "libjpeg-dev:armhf", + "libfreetype6-dev:armhf", + "libevdev-dev:armhf", + ), + ), +} +TARGET_CROSS_TOOLCHAIN_PACKAGES = tuple( + dict.fromkeys( + package + for toolchain in TARGET_CROSS_TOOLCHAINS.values() + for package in toolchain.packages + ) +) +TARGET_CROSS_TOOLCHAIN_DPKG_ARCHS = tuple( + dict.fromkeys(toolchain.dpkg_arch for toolchain in TARGET_CROSS_TOOLCHAINS.values()) +) diff --git a/backend/app/utils/local_exec.py b/backend/app/utils/local_exec.py index 4a7da4ca4..f815ea080 100644 --- a/backend/app/utils/local_exec.py +++ b/backend/app/utils/local_exec.py @@ -1,10 +1,14 @@ -from arq import ArqRedis -import asyncio +from __future__ import annotations + from typing import Literal, Optional, Tuple + +from arq import ArqRedis from sqlalchemy.orm import Session -from app.models.log import new_log as log from app.models.frame import Frame +from app.utils.build_executor import create_build_executor +from app.utils.modal_sandbox import get_modal_sandbox_config + async def exec_local_command( db: Session | None, @@ -15,79 +19,20 @@ async def exec_local_command( log_output: bool = True, stderr_log_tag: Literal["stderr", "stdout"] = "stderr", ) -> Tuple[int, Optional[str], Optional[str]]: - - if log_command: - if db and redis: - await log(db, redis, int(frame.id), "stdout", f"$ {log_command if isinstance(log_command, str) else command}") - else: - print(f"$ {log_command if isinstance(log_command, str) else command}") - - proc = await asyncio.create_subprocess_shell( - command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - - async def pump(stream, tag, buf): - pending = "" - - async def _flush(segment: str, *, terminated: bool): - if not segment: - return - buf.append(f"{segment}\n" if terminated else segment) - if log_output: - if db and redis: - await log(db, redis, int(frame.id), tag, segment) - else: - print(segment) - - while True: - raw = await stream.read(1024) - if not raw: # EOF - break - - chunk = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw) - pending += chunk - - while True: - newline_index = pending.find("\n") - carriage_index = pending.find("\r") - - split_index = -1 - if newline_index != -1 and carriage_index != -1: - split_index = min(newline_index, carriage_index) - elif newline_index != -1: - split_index = newline_index - elif carriage_index != -1: - split_index = carriage_index - - if split_index == -1: - break - - segment = pending[:split_index].strip("\r") - pending = pending[split_index + 1 :] - await _flush(segment, terminated=True) - - pending = pending.strip("\r") - if pending: - await _flush(pending, terminated=False) - - out_buf: list[str] = [] - err_buf: list[str] = [] - await asyncio.gather( - pump(proc.stdout, "stdout", out_buf), - pump(proc.stderr, stderr_log_tag, err_buf), - ) - - exit_code = await proc.wait() - if exit_code: - if db and redis: - await log(db, redis, int(frame.id), "exit_status", f"The command exited with status {exit_code}") - else: - print( f"The command exited with status {exit_code}") - - return ( - exit_code, - "".join(out_buf) or None, - "".join(err_buf) or None, + project_id = getattr(frame, "project_id", None) + # Keep this compatibility helper scoped to local/Modal execution. Build-host + # commands need structured path syncing via explicit BuildExecutor use. + modal_config = get_modal_sandbox_config(db, project_id) if project_id is not None else None + executor = create_build_executor( + modal_config, + db=db, + redis=redis, + frame=frame, ) + async with executor: + return await executor.run( + command, + log_command=log_command, + log_output=log_output, + stderr_log_tag=stderr_log_tag, + ) diff --git a/backend/app/utils/modal_sandbox.py b/backend/app/utils/modal_sandbox.py new file mode 100644 index 000000000..8c4b0702e --- /dev/null +++ b/backend/app/utils/modal_sandbox.py @@ -0,0 +1,734 @@ +from __future__ import annotations + +import asyncio +import contextlib +import inspect +import os +import re +import shlex +import tarfile +import tempfile +from dataclasses import dataclass +from pathlib import Path +from typing import Awaitable, Callable + +from sqlalchemy.orm import Session + +from app.models.settings import get_settings_dict +from app.utils.build_environment import selected_build_environment_provider + +LogFunc = Callable[[str, str], Awaitable[None]] + +REPO_ROOT = Path(__file__).resolve().parents[3] +DEFAULT_MODAL_APP_NAME = os.environ.get("FRAMEOS_MODAL_SANDBOX_APP", "frameos-build") +DEFAULT_MODAL_IMAGE = os.environ.get("FRAMEOS_MODAL_SANDBOX_IMAGE", "frameos/frameos:latest") +DEFAULT_MODAL_TIMEOUT = int(os.environ.get("FRAMEOS_MODAL_SANDBOX_TIMEOUT", str(6 * 60 * 60))) +DEFAULT_MODAL_IDLE_TIMEOUT = int(os.environ.get("FRAMEOS_MODAL_SANDBOX_IDLE_TIMEOUT", "60")) +FRAMEOS_SANDBOX_PATH = "/opt/nim/bin:/root/.nimble/bin:/app/backend/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +PATH_TOKEN_RE = re.compile(r"(? ModalSandboxConfig | None: + if not isinstance(raw, dict): + return None + enabled = bool(raw.get("enabled")) + token_id = str(raw.get("tokenId") or raw.get("token_id") or "").strip() + token_secret = str(raw.get("tokenSecret") or raw.get("token_secret") or "").strip() + if not enabled or not token_id or not token_secret: + return None + + def optional_int(key: str, default: int) -> int: + try: + return int(raw.get(key) or default) + except (TypeError, ValueError): + return default + + def optional_float(key: str) -> float | None: + value = raw.get(key) + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + return cls( + token_id=token_id, + token_secret=token_secret, + app_name=str(raw.get("appName") or raw.get("app_name") or DEFAULT_MODAL_APP_NAME).strip() + or DEFAULT_MODAL_APP_NAME, + image=str(raw.get("image") or DEFAULT_MODAL_IMAGE).strip() or DEFAULT_MODAL_IMAGE, + enabled=True, + timeout=optional_int("timeout", DEFAULT_MODAL_TIMEOUT), + idle_timeout=optional_int("idleTimeout", optional_int("idle_timeout", DEFAULT_MODAL_IDLE_TIMEOUT)), + cpu=optional_float("cpu"), + memory=optional_int("memory", 0) or None, + region=str(raw.get("region") or "").strip() or None, + cloud=str(raw.get("cloud") or "").strip() or None, + environment_name=str(raw.get("environmentName") or raw.get("environment_name") or "").strip() or None, + enable_docker=raw.get("enableDocker", raw.get("enable_docker", True)) is not False, + ) + + +@dataclass(slots=True) +class DockerRunMount: + source: Path + target: str + read_only: bool = False + + +@dataclass(slots=True) +class DockerRunSpec: + image: str + args: list[str] + mounts: list[DockerRunMount] + env: dict[str, str] + workdir: str | None = None + platform: str | None = None + + +def get_modal_sandbox_config(db: Session | None, project_id: int | None = None) -> ModalSandboxConfig | None: + if db is None or project_id is None: + return None + settings = get_settings_dict(db, project_id=project_id) + if selected_build_environment_provider(settings) != "modal": + return None + return ModalSandboxConfig.from_settings(settings.get("modalSandbox")) + + +def parse_docker_run_command(command: str) -> DockerRunSpec | None: + try: + tokens = shlex.split(command) + except ValueError: + return None + if len(tokens) < 3 or tokens[0:2] != ["docker", "run"]: + return None + + mounts: list[DockerRunMount] = [] + env: dict[str, str] = {} + workdir: str | None = None + platform: str | None = None + index = 2 + while index < len(tokens): + token = tokens[index] + if token == "--": + index += 1 + break + if token in {"--rm", "-i", "-t", "-it", "--init"}: + index += 1 + continue + if token in {"-v", "--volume"} and index + 1 < len(tokens): + mount = _parse_docker_mount(tokens[index + 1]) + if mount: + mounts.append(mount) + index += 2 + continue + if token.startswith("--volume="): + mount = _parse_docker_mount(token.split("=", 1)[1]) + if mount: + mounts.append(mount) + index += 1 + continue + if token.startswith("-v") and len(token) > 2: + mount = _parse_docker_mount(token[2:]) + if mount: + mounts.append(mount) + index += 1 + continue + if token in {"-e", "--env"} and index + 1 < len(tokens): + _add_docker_env(env, tokens[index + 1]) + index += 2 + continue + if token.startswith("--env="): + _add_docker_env(env, token.split("=", 1)[1]) + index += 1 + continue + if token in {"-w", "--workdir"} and index + 1 < len(tokens): + workdir = tokens[index + 1] + index += 2 + continue + if token.startswith("--workdir="): + workdir = token.split("=", 1)[1] + index += 1 + continue + if token == "--platform" and index + 1 < len(tokens): + platform = tokens[index + 1] + index += 2 + continue + if token.startswith("--platform="): + platform = token.split("=", 1)[1] + index += 1 + continue + if token in {"--ulimit", "--name", "--user", "-u", "--entrypoint"} and index + 1 < len(tokens): + index += 2 + continue + if token.startswith("--ulimit=") or token.startswith("--name=") or token.startswith("--user=") or token.startswith("--entrypoint="): + index += 1 + continue + if token.startswith("-"): + return None + image = token + return DockerRunSpec(image=image, args=tokens[index + 1 :], mounts=mounts, env=env, workdir=workdir, platform=platform) + return None + + +def _parse_docker_mount(value: str) -> DockerRunMount | None: + parts = value.split(":") + if len(parts) < 2 or not parts[0].startswith("/") or not parts[1].startswith("/"): + return None + options = set(parts[2].split(",")) if len(parts) > 2 else set() + return DockerRunMount(source=Path(parts[0]), target=parts[1], read_only="ro" in options) + + +def _add_docker_env(env: dict[str, str], value: str) -> None: + if "=" in value: + key, env_value = value.split("=", 1) + if key: + env[key] = env_value + elif value in os.environ: + env[value] = os.environ[value] + + +async def _maybe_await(value): + if hasattr(value, "__await__"): + return await value + return value + + +async def _call_modal(method, *args, **kwargs): + aio = getattr(method, "aio", None) + if aio is not None: + return await aio(*args, **kwargs) + result = method(*args, **kwargs) + if inspect.isawaitable(result): + return await result + return result + + +class _FrameOSModalOutputManager: + def __init__(self, base_manager, logger: LogFunc | None, loop: asyncio.AbstractEventLoop | None = None) -> None: + self._base_manager = base_manager + self._logger = logger + try: + self._loop = loop or asyncio.get_running_loop() + except RuntimeError: + self._loop = loop + self._pending: dict[int, str] = {} + self.logged_lines = 0 + + @property + def is_enabled(self) -> bool: + return True + + @property + def is_terminal(self) -> bool: + return False + + @property + def _show_image_logs(self) -> bool: + return True + + def __getattr__(self, name: str): + return getattr(self._base_manager, name) + + def enable_image_logs(self) -> None: + return None + + async def put_streaming_log(self, log, prefix: str = "") -> None: # noqa: ANN001 + await self._write_log(log, prefix=prefix) + + async def put_fetched_log(self, log, prefix: str = "") -> None: # noqa: ANN001 + await self._write_log(log, prefix=prefix) + + async def _write_log(self, log, prefix: str = "") -> None: # noqa: ANN001 + if self._logger is None: + return + fd = int(getattr(log, "file_descriptor", 1) or 1) + tag = "stderr" if fd == 2 else "stdout" + text = str(getattr(log, "data", "") or "") + if prefix: + text = prefix + text + pending = self._pending.get(fd, "") + text + lines = pending.splitlines(keepends=True) + self._pending[fd] = "" + if lines and not lines[-1].endswith(("\n", "\r")): + self._pending[fd] = lines.pop() + for line in lines: + message = line.rstrip("\r\n") + if message: + self.logged_lines += 1 + await self._emit_log(tag, f"Modal image build: {message}") + + async def flush_pending(self) -> None: + if self._logger is None: + self._pending.clear() + return + for fd, pending in list(self._pending.items()): + message = pending.rstrip("\r\n") + if message: + tag = "stderr" if fd == 2 else "stdout" + self.logged_lines += 1 + await self._emit_log(tag, f"Modal image build: {message}") + self._pending.clear() + + async def _emit_log(self, tag: str, message: str) -> None: + if self._logger is None: + return + if self._loop is None: + await self._logger(tag, message) + return + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if running_loop is self._loop: + await self._logger(tag, message) + return + future = asyncio.run_coroutine_threadsafe(self._logger(tag, message), self._loop) + await asyncio.wrap_future(future) + + +@contextlib.contextmanager +def _modal_output_to_logs(modal, logger: LogFunc | None): # noqa: ANN001 + try: + from modal.output import OutputManager + except Exception: + yield None + return + previous = OutputManager.get() + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + manager = _FrameOSModalOutputManager(previous, logger, loop=loop) + OutputManager._set(manager) + try: + yield manager + finally: + OutputManager._set(previous) + + +def _is_timeout_error(exc: Exception) -> bool: + if isinstance(exc, TimeoutError): + return True + name = exc.__class__.__name__.lower() + message = str(exc).lower() + return "timeout" in name or "timed out" in message or "timeout" in message + + +class ModalSandboxSession: + def __init__(self, config: ModalSandboxConfig, *, logger: LogFunc | None = None) -> None: + self.config = config + self._logger = logger + self._modal = None + self._client = None + self._sandbox = None + self._cleanup_paths: list[str] = [] + + async def __aenter__(self) -> "ModalSandboxSession": + await self._connect() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: # noqa: ANN001 + if self._sandbox: + try: + await _call_modal(self._sandbox.terminate, wait=True) + except Exception: + pass + try: + await _call_modal(self._sandbox.detach) + except Exception: + pass + + async def _connect(self) -> None: + try: + import modal + except ImportError as exc: # pragma: no cover - depends on deployment deps + raise RuntimeError("Modal sandbox execution requires the 'modal' Python package") from exc + + self._modal = modal + self._client = await _call_modal(modal.Client.from_credentials, self.config.token_id, self.config.token_secret) + app = await _call_modal( + modal.App.lookup, + self.config.app_name, + client=self._client, + environment_name=self.config.environment_name, + create_if_missing=True, + ) + image = ( + modal.Image.from_registry( + self.config.image, + setup_dockerfile_commands=[ + f"ENV PATH={FRAMEOS_SANDBOX_PATH}", + ( + "RUN if ! command -v python >/dev/null 2>&1 " + "&& command -v python3 >/dev/null 2>&1; then " + "ln -sf \"$(command -v python3)\" /usr/local/bin/python; fi" + ), + ], + ) + if self.config.image + else None + ) + kwargs = { + "app": app, + "client": self._client, + "image": image, + "env": {"PATH": FRAMEOS_SANDBOX_PATH, "HOME": "/root"}, + "timeout": self.config.timeout, + "idle_timeout": self.config.idle_timeout, + "cpu": self.config.cpu, + "memory": self.config.memory, + "region": self.config.region, + "cloud": self.config.cloud, + "environment_name": self.config.environment_name, + "experimental_options": {"enable_docker": True} if self.config.enable_docker else None, + } + kwargs = {key: value for key, value in kwargs.items() if value is not None} + with _modal_output_to_logs(modal, self._logger) as output_manager: + try: + self._sandbox = await _call_modal(modal.Sandbox.create, "sleep", str(self.config.timeout), **kwargs) + except Exception as exc: + if output_manager is not None: + await output_manager.flush_pending() + if _is_timeout_error(exc): + await self._log_timeout_notice("creating Modal sandbox") + elif output_manager is not None and output_manager.logged_lines == 0: + await self._log( + "stderr", + "Modal sandbox image build failed before Modal returned build-log lines. " + f"app={self.config.app_name}, image={self.config.image}, " + f"region={self.config.region or 'auto'}, cloud={self.config.cloud or 'auto'}. " + "Check the Modal dashboard for the image build record if the SDK error remains generic.", + ) + raise + finally: + if output_manager is not None: + await output_manager.flush_pending() + identity = await self._sandbox_identity() + await self._log("stdout", self._connection_summary(identity)) + + async def _sandbox_identity(self) -> dict[str, str]: + if not self._sandbox: + return {} + command = ( + "printf 'host=%s\\n' \"$(hostname 2>/dev/null || printf unknown)\"; " + "printf 'cpu_count=%s\\n' \"$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || printf unknown)\"; " + "awk '/MemTotal/ {printf \"memory_mib=%d\\n\", $2 / 1024}' /proc/meminfo 2>/dev/null || true" + ) + try: + proc = await _call_modal(self._sandbox.exec, "bash", "-lc", command, timeout=min(self.config.timeout, 30)) + out_buf: list[str] = [] + async for chunk in proc.stdout: + out_buf.append(chunk.decode("utf-8", errors="replace") if isinstance(chunk, (bytes, bytearray)) else str(chunk)) + async for _chunk in proc.stderr: + pass + await _call_modal(proc.wait) + except Exception: + return {} + identity: dict[str, str] = {} + for raw_line in "".join(out_buf).splitlines(): + key, separator, value = raw_line.partition("=") + if separator and key and value: + identity[key.strip()] = value.strip() + return identity + + def _connection_summary(self, identity: dict[str, str]) -> str: + sandbox_id = "" + if self._sandbox: + for attr in ("object_id", "sandbox_id", "id", "_object_id"): + value = getattr(self._sandbox, attr, None) + if value: + sandbox_id = str(value) + break + cpu = f"{self.config.cpu:g} requested" if self.config.cpu is not None else "auto requested" + memory = f"{self.config.memory} MiB requested" if self.config.memory is not None else "auto requested" + parts = [ + "Connected to Modal sandbox", + f"app={self.config.app_name}", + f"environment={self.config.environment_name or 'default'}", + f"sandbox={sandbox_id or 'unknown'}", + f"host={identity.get('host') or 'unknown'}", + f"image={self.config.image}", + f"cpu={cpu}", + f"memory={memory}", + f"region={self.config.region or 'auto'}", + f"cloud={self.config.cloud or 'auto'}", + f"timeout={self.config.timeout}s", + f"idle_timeout={self.config.idle_timeout}s", + f"nested_docker={'enabled' if self.config.enable_docker else 'disabled'}", + ] + return f"{parts[0]}: " + ", ".join(parts[1:]) + + async def _log(self, level: str, message: str) -> None: + if self._logger: + await self._logger(level, message) + + async def _log_timeout_notice(self, action: str) -> None: + await self._log( + "stderr", + "Modal sandbox timed out while " + f"{action}. Configured timeout={self.config.timeout}s, " + f"idle_timeout={self.config.idle_timeout}s, app={self.config.app_name}, " + f"image={self.config.image}. Increase the Modal sandbox timeout/idle timeout " + "in global settings if this build legitimately needs longer.", + ) + + async def run( + self, + command: str, + *, + log_command: str | bool = True, + log_output: bool = True, + ) -> tuple[int, str | None, str | None]: + if not self._sandbox: + raise RuntimeError("Modal sandbox session is not connected") + + if log_command: + await self._log("stdout", f"$ {log_command if isinstance(log_command, str) else command}") + + wrapped_command = f"export PATH={shlex.quote(FRAMEOS_SANDBOX_PATH)} HOME=/root; {command}" + try: + proc = await _call_modal(self._sandbox.exec, "bash", "-lc", wrapped_command, timeout=self.config.timeout) + out_buf: list[str] = [] + err_buf: list[str] = [] + + async def pump(stream, level: str, buf: list[str]) -> None: + pending = "" + async for chunk in stream: + text = chunk.decode("utf-8", errors="replace") if isinstance(chunk, (bytes, bytearray)) else str(chunk) + pending += text + while True: + split_index = pending.find("\n") + if split_index == -1: + break + segment = pending[:split_index].rstrip("\r") + pending = pending[split_index + 1 :] + buf.append(f"{segment}\n") + if log_output and segment: + await self._log(level, segment) + pending = pending.rstrip("\r") + if pending: + buf.append(pending) + if log_output: + await self._log(level, pending) + + await asyncio.gather(pump(proc.stdout, "stdout", out_buf), pump(proc.stderr, "stderr", err_buf)) + return_code = await _call_modal(proc.wait) + except Exception as exc: + if _is_timeout_error(exc): + await self._log_timeout_notice("running Modal command") + raise + if return_code and log_output: + await self._log("exit_status", f"The command exited with status {return_code}") + return int(return_code or 0), "".join(out_buf) or None, "".join(err_buf) or None + + async def mktemp_dir(self, prefix: str = "frameos-build-") -> str: + status, out, _err = await self.run(f"mktemp -d -p /tmp {prefix}XXXXXX", log_command=False, log_output=False) + if status != 0 or not out: + raise RuntimeError("Failed to allocate temporary directory in Modal sandbox") + path = out.strip().splitlines()[-1] + self._cleanup_paths.append(path) + return path + + async def ensure_dir(self, remote_path: str) -> None: + await self.run(f"mkdir -p {shlex.quote(remote_path)}", log_command=False, log_output=False) + + async def remove_path(self, remote_path: str) -> None: + await self.run(f"rm -rf {shlex.quote(remote_path)}", log_command=False, log_output=False) + + async def sync_dir_tarball(self, local_path: str, remote_path: str) -> None: + local = Path(local_path) + if not local.exists(): + return + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp: + archive_path = Path(tmp.name) + try: + with tarfile.open(archive_path, "w:gz") as tar: + tar.add(local, arcname=".") + await self.sync_dir_archive(str(archive_path), remote_path) + finally: + archive_path.unlink(missing_ok=True) + + async def sync_dir_archive(self, archive_path: str, remote_path: str) -> None: + remote_archive = f"{remote_path}.tar.gz" + await self.ensure_dir(str(Path(remote_path).parent)) + await self.remove_path(remote_path) + await self._copy_from_local(archive_path, remote_archive) + status, _out, err = await self.run( + " ".join( + [ + "mkdir -p", + shlex.quote(remote_path), + "&& tar -xzf", + shlex.quote(remote_archive), + "-C", + shlex.quote(remote_path), + "&& rm -f", + shlex.quote(remote_archive), + ] + ), + log_command=False, + log_output=False, + ) + if status != 0: + raise RuntimeError(f"Failed to extract Modal sandbox archive: {err or 'see logs'}") + + async def sync_file(self, local_path: str, remote_path: str) -> None: + if not Path(local_path).is_file(): + return + await self.ensure_dir(str(Path(remote_path).parent)) + await self._copy_from_local(local_path, remote_path) + + async def write_file(self, remote_path: str, content: str, mode: int = 0o644) -> None: + await self.ensure_dir(str(Path(remote_path).parent)) + await _call_modal(self._sandbox.filesystem.write_text, content, remote_path) + await self.run(f"chmod {oct(mode)[2:]} {shlex.quote(remote_path)}", log_command=False, log_output=False) + + async def download_file(self, remote_path: str, local_path: str) -> None: + Path(local_path).parent.mkdir(parents=True, exist_ok=True) + await _call_modal(self._sandbox.filesystem.copy_to_local, remote_path, local_path) + + async def download_dir_tarball(self, remote_path: str, local_path: str) -> None: + local = Path(local_path) + with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp: + archive_path = Path(tmp.name) + remote_archive = f"/tmp/frameos-download-{os.getpid()}-{abs(hash(remote_path))}.tar.gz" + try: + status, _out, err = await self.run( + f"test -e {shlex.quote(remote_path)} && tar -czf {shlex.quote(remote_archive)} -C {shlex.quote(remote_path)} .", + log_command=False, + log_output=False, + ) + if status != 0: + return + await self.download_file(remote_archive, str(archive_path)) + if local.exists() and local.is_dir(): + for child in local.iterdir(): + if child.is_dir(): + import shutil + + shutil.rmtree(child) + else: + child.unlink() + local.mkdir(parents=True, exist_ok=True) + with tarfile.open(archive_path, "r:gz") as tar: + tar.extractall(local) + finally: + archive_path.unlink(missing_ok=True) + await self.remove_path(remote_archive) + + async def _copy_from_local(self, local_path: str, remote_path: str) -> None: + await _call_modal(self._sandbox.filesystem.copy_from_local, local_path, remote_path) + + +def _path_is_allowed(path: Path) -> bool: + try: + resolved = path.resolve() + except OSError: + return False + resolved_str = str(resolved) + if any(resolved_str == prefix or resolved_str.startswith(f"{prefix}/") for prefix in SKIP_PATH_PREFIXES): + return False + if resolved_str == "/tmp" or resolved_str == "/private/tmp": + return False + if resolved_str.startswith(str(REPO_ROOT)): + return True + tmpdir = os.environ.get("TMPDIR") + if tmpdir and resolved_str.startswith(str(Path(tmpdir).resolve())): + return True + return resolved_str.startswith("/tmp/") or resolved_str.startswith("/private/tmp/") + + +def _docker_mount_sources(command: str) -> set[Path]: + paths: set[Path] = set() + try: + tokens = shlex.split(command) + except ValueError: + return paths + for index, token in enumerate(tokens): + value = "" + if token in {"-v", "--volume"} and index + 1 < len(tokens): + value = tokens[index + 1] + elif token.startswith("-v") and len(token) > 2: + value = token[2:] + elif token.startswith("--volume="): + value = token.split("=", 1)[1] + if not value: + continue + source = value.split(":", 1)[0] + if source.startswith("/"): + paths.add(Path(source)) + return paths + + +def _absolute_paths(command: str) -> set[Path]: + paths: set[Path] = set() + for match in PATH_TOKEN_RE.finditer(command): + raw = match.group(0).rstrip(".,;:)") + if raw: + paths.add(Path(raw)) + return paths + + +def sandbox_sync_paths_for_command(command: str) -> list[Path]: + candidates = _docker_mount_sources(command) | _absolute_paths(command) + existing: list[tuple[Path, Path]] = [] + for candidate in candidates: + path = candidate + while not path.exists() and path != path.parent: + path = path.parent + if not path.exists() or not _path_is_allowed(path): + continue + try: + resolved = path.resolve() + except OSError: + continue + if any( + resolved == existing_resolved or str(resolved).startswith(f"{existing_resolved}/") + for _existing_path, existing_resolved in existing + ): + continue + existing = [ + (existing_path, existing_resolved) + for existing_path, existing_resolved in existing + if not str(existing_resolved).startswith(f"{resolved}/") + ] + existing.append((path, resolved)) + return [path for path, _resolved in sorted(existing, key=lambda item: len(str(item[0])))] diff --git a/backend/app/utils/system_info.py b/backend/app/utils/system_info.py index 36228b3b9..a22025a13 100644 --- a/backend/app/utils/system_info.py +++ b/backend/app/utils/system_info.py @@ -52,6 +52,13 @@ class DatabaseUsage: exists: bool +@dataclass(frozen=True) +class DockerInfo: + cli_available: bool + daemon_available: bool + error: str | None = None + + def _safe_stat(path: Path) -> int: try: return path.stat().st_size @@ -174,13 +181,36 @@ def get_cache_usage() -> list[CacheUsage]: return caches -def get_system_info() -> tuple[DiskUsage, list[CacheUsage], DatabaseUsage, MemoryInfo, LoadAverage]: +def get_docker_info() -> DockerInfo: + if shutil.which("docker") is None: + return DockerInfo(cli_available=False, daemon_available=False, error="Docker CLI is not installed") + try: + result = subprocess.run( + ["docker", "info"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + except Exception as exc: # noqa: BLE001 + return DockerInfo(cli_available=True, daemon_available=False, error=str(exc)) + if result.returncode == 0: + return DockerInfo(cli_available=True, daemon_available=True) + return DockerInfo( + cli_available=True, + daemon_available=False, + error=(result.stderr or result.stdout or "Docker daemon is not reachable").strip(), + ) + + +def get_system_info() -> tuple[DiskUsage, list[CacheUsage], DatabaseUsage, MemoryInfo, LoadAverage, DockerInfo]: disk = get_disk_usage() caches = get_cache_usage() database = get_database_usage() memory = get_memory_info() load = get_load_average() - return disk, caches, database, memory, load + docker = get_docker_info() + return disk, caches, database, memory, load, docker def get_system_metrics() -> tuple[DiskUsage, MemoryInfo, LoadAverage]: diff --git a/backend/app/utils/tests/test_build_environment.py b/backend/app/utils/tests/test_build_environment.py new file mode 100644 index 000000000..6d916083a --- /dev/null +++ b/backend/app/utils/tests/test_build_environment.py @@ -0,0 +1,28 @@ +from app.utils.build_environment import selected_build_environment_provider + + +def test_selected_build_environment_defaults_to_docker(): + assert selected_build_environment_provider({}) == "docker" + + +def test_selected_build_environment_uses_explicit_provider(): + assert selected_build_environment_provider({"buildEnvironment": {"provider": "none"}}) == "none" + assert selected_build_environment_provider({"buildEnvironment": {"provider": "modal"}}) == "modal" + + +def test_selected_build_environment_infers_legacy_settings(): + assert selected_build_environment_provider({"modalSandbox": {"enabled": True}}) == "modal" + assert selected_build_environment_provider({"buildHost": {"enabled": True}}) == "buildHost" + + +def test_selected_build_environment_explicit_provider_wins_over_legacy_flags(): + assert ( + selected_build_environment_provider( + { + "buildEnvironment": {"provider": "docker"}, + "modalSandbox": {"enabled": True}, + "buildHost": {"enabled": True}, + } + ) + == "docker" + ) diff --git a/backend/app/utils/tests/test_build_executor.py b/backend/app/utils/tests/test_build_executor.py new file mode 100644 index 000000000..beb3e1e75 --- /dev/null +++ b/backend/app/utils/tests/test_build_executor.py @@ -0,0 +1,196 @@ +from pathlib import Path + +import pytest + +from app.utils.build_executor import DockerMount, ModalBuildExecutor +from app.utils.modal_sandbox import ModalSandboxConfig + + +@pytest.mark.asyncio +async def test_modal_executor_routes_docker_run_through_direct_image(monkeypatch, tmp_path): + work = tmp_path / "work" + work.mkdir() + calls = [] + + class FakeModalSandboxSession: + def __init__(self, config, logger=None): + self.config = config + self.logger = logger + calls.append(("init", config)) + + async def __aenter__(self): + calls.append(("enter",)) + return self + + async def __aexit__(self, exc_type, exc, tb): + calls.append(("exit", exc_type)) + return None + + async def sync_dir_tarball(self, local_path, remote_path): + calls.append(("sync_dir", Path(local_path), remote_path)) + + async def sync_dir_archive(self, archive_path, remote_path): + calls.append(("sync_archive", Path(archive_path), remote_path)) + + async def sync_file(self, local_path, remote_path): + calls.append(("sync_file", Path(local_path), remote_path)) + + async def download_dir_tarball(self, remote_path, local_path): + calls.append(("download_dir", remote_path, Path(local_path))) + + async def download_file(self, remote_path, local_path): + calls.append(("download_file", remote_path, Path(local_path))) + + async def run(self, command, **kwargs): + calls.append(("run", command, kwargs)) + return 0, "ok\n", None + + monkeypatch.setattr("app.utils.build_executor.ModalSandboxSession", FakeModalSandboxSession) + + executor = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + enable_docker=True, + ) + ) + + status, out, err = await executor.run( + f"docker run --rm -v {work}:/work -w /work example/image:latest bash build.sh", + log_command=False, + log_output=False, + ) + + assert (status, out, err) == (0, "ok\n", None) + assert calls[0][0] == "init" + config = calls[0][1] + assert config.image == "example/image:latest" + assert config.enable_docker is False + assert config.cpu == 2 + assert config.memory == 4096 + sync_archive_calls = [call for call in calls if call[0] == "sync_archive"] + assert len(sync_archive_calls) == 1 + assert sync_archive_calls[0][1].suffixes[-2:] == [".tar", ".gz"] + assert sync_archive_calls[0][2] == "/work" + assert ( + "run", + "cd /work && bash build.sh", + {"log_command": False, "log_output": False}, + ) in calls + assert ("download_dir", "/work", work) in calls + + +def test_modal_executor_uses_plain_tag_for_direct_container_images(): + executor = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + ) + ) + + image = "frameos/frameos-cross-toolchain:debian_bookworm-linux_arm64-latest" + resolved = f"{image}@sha256:17e810a39fb457429852a63587d39a7dcdee436771d02c50254b4f98976c1e38" + + assert executor.container_image_reference(image, resolved) == image + + +@pytest.mark.asyncio +async def test_modal_executor_rejects_non_amd64_direct_container_platform(monkeypatch): + calls = [] + + class FakeModalSandboxSession: + def __init__(self, config, logger=None): + self.config = config + self.logger = logger + calls.append(("init", config)) + + async def __aenter__(self): + calls.append(("enter",)) + return self + + async def __aexit__(self, exc_type, exc, tb): + calls.append(("exit", exc_type)) + return None + + monkeypatch.setattr("app.utils.build_executor.ModalSandboxSession", FakeModalSandboxSession) + + logs = [] + + async def logger(level, message): + logs.append((level, message)) + + executor = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + enable_docker=False, + ), + logger=logger, + ) + + status, out, err = await executor.docker_run( + image="frameos/frameos-cross-toolchain:debian_bookworm-linux_arm64-latest", + platform="linux/arm64", + mounts=[], + workdir="/src", + args=["bash", "build.sh"], + log_command=False, + log_output=False, + ) + + assert status == 125 + assert out is None + assert err is not None + assert "linux/amd64 registry images" in err + assert calls == [] + assert logs == [("stderr", err.strip())] + + +def test_modal_executor_maps_non_amd64_targets_to_amd64_container_platform(): + executor = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + ) + ) + + assert executor.container_platform_for_target("linux/arm64") == "linux/amd64" + assert executor.container_platform_for_target("linux/amd64") == "linux/amd64" + + +def test_modal_executor_resource_profiles_respect_user_overrides(): + executor = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + ) + ) + + cross_compile = executor._config_with_resources(image="toolchain", workspace="cross-compile") + compose = executor._config_with_resources(image="buildroot", workspace="compose") + + assert (cross_compile.cpu, cross_compile.memory) == (8, 16384) + assert (compose.cpu, compose.memory) == (4, 8192) + + overridden = ModalBuildExecutor( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:base", + cpu=6, + memory=12288, + ) + )._config_with_resources(image="toolchain", workspace="cross-compile") + + assert (overridden.cpu, overridden.memory) == (6, 12288) diff --git a/backend/app/utils/tests/test_build_host.py b/backend/app/utils/tests/test_build_host.py index 176c1810b..b258ec2d2 100644 --- a/backend/app/utils/tests/test_build_host.py +++ b/backend/app/utils/tests/test_build_host.py @@ -3,7 +3,8 @@ import pytest from app.models.settings import Settings -from app.utils.build_host import BuildHostConfig, BuildHostSession, get_build_host_config +from app.utils.build_host import BuildHostConfig, BuildHostSession, get_build_executor_config, get_build_host_config +from app.utils.modal_sandbox import ModalSandboxConfig @pytest.mark.asyncio @@ -52,6 +53,74 @@ async def test_get_build_host_config_requires_fields(db, default_project): assert get_build_host_config(db, default_project.id) is None +@pytest.mark.asyncio +async def test_get_build_executor_config_prefers_modal_sandbox(db, default_project): + db.query(Settings).delete() + db.add( + Settings( + project_id=default_project.id, + key="buildHost", + value={ + "enabled": True, + "host": "builder.local", + "user": "ubuntu", + "sshKey": "dummy-key", + }, + ) + ) + db.add( + Settings( + project_id=default_project.id, + key="modalSandbox", + value={ + "enabled": True, + "tokenId": "ak-test", + "tokenSecret": "as-test", + "appName": "frameos-test", + "image": "frameos/frameos:test", + }, + ) + ) + db.commit() + + config = get_build_executor_config(db, default_project.id) + assert isinstance(config, ModalSandboxConfig) + assert config.app_name == "frameos-test" + assert config.image == "frameos/frameos:test" + + +@pytest.mark.asyncio +async def test_get_build_executor_config_uses_selected_provider(db, default_project): + db.query(Settings).delete() + db.add(Settings(project_id=default_project.id, key="buildEnvironment", value={"provider": "docker"})) + db.add( + Settings( + project_id=default_project.id, + key="buildHost", + value={ + "enabled": True, + "host": "builder.local", + "user": "ubuntu", + "sshKey": "dummy-key", + }, + ) + ) + db.add( + Settings( + project_id=default_project.id, + key="modalSandbox", + value={ + "enabled": True, + "tokenId": "ak-test", + "tokenSecret": "as-test", + }, + ) + ) + db.commit() + + assert get_build_executor_config(db, default_project.id) is None + + @pytest.mark.asyncio async def test_build_host_run_preserves_buffered_stdout_line_breaks(): class FakeStream: diff --git a/backend/app/utils/tests/test_local_exec.py b/backend/app/utils/tests/test_local_exec.py index 89982adf4..63631502d 100644 --- a/backend/app/utils/tests/test_local_exec.py +++ b/backend/app/utils/tests/test_local_exec.py @@ -28,7 +28,7 @@ async def test_exec_local_command_can_log_stderr_as_stdout(monkeypatch): async def fake_log(_db, _redis, frame_id, tag, message): entries.append((frame_id, tag, message)) - monkeypatch.setattr("app.utils.local_exec.log", fake_log) + monkeypatch.setattr("app.utils.build_executor.log", fake_log) status, out, err = await exec_local_command( object(), @@ -46,3 +46,52 @@ async def fake_log(_db, _redis, frame_id, tag, message): (1, "stdout", "out"), (1, "stdout", "err"), ] + + +@pytest.mark.asyncio +async def test_exec_local_command_runs_through_build_executor(monkeypatch): + calls = [] + + class FakeExecutor: + async def __aenter__(self): + calls.append(("enter",)) + return self + + async def __aexit__(self, exc_type, exc, tb): + calls.append(("exit", exc_type)) + return None + + async def run(self, command, **kwargs): + calls.append(("run", command, kwargs)) + return 0, "ok\n", None + + def fake_create_build_executor(config, **kwargs): + calls.append(("factory", config, kwargs)) + return FakeExecutor() + + monkeypatch.setattr("app.utils.local_exec.get_modal_sandbox_config", lambda db, project_id: None) + monkeypatch.setattr("app.utils.local_exec.create_build_executor", fake_create_build_executor) + + status, out, err = await exec_local_command( + object(), + object(), + SimpleNamespace(id=1, project_id=2), + "echo ok", + log_command=False, + log_output=False, + stderr_log_tag="stdout", + ) + + assert (status, out, err) == (0, "ok\n", None) + assert calls[0][0] == "factory" + assert calls[0][1] is None + assert calls[0][2]["frame"].id == 1 + assert calls[1:] == [ + ("enter",), + ( + "run", + "echo ok", + {"log_command": False, "log_output": False, "stderr_log_tag": "stdout"}, + ), + ("exit", None), + ] diff --git a/backend/app/utils/tests/test_modal_sandbox.py b/backend/app/utils/tests/test_modal_sandbox.py new file mode 100644 index 000000000..343dab5e7 --- /dev/null +++ b/backend/app/utils/tests/test_modal_sandbox.py @@ -0,0 +1,329 @@ +import contextlib +from pathlib import Path +import sys +from types import SimpleNamespace + +import pytest + +from app.utils.modal_sandbox import ( + FRAMEOS_SANDBOX_PATH, + ModalSandboxConfig, + ModalSandboxSession, + _FrameOSModalOutputManager, + parse_docker_run_command, + sandbox_sync_paths_for_command, +) + + +def test_modal_sandbox_config_requires_credentials(): + assert ModalSandboxConfig.from_settings({"enabled": True, "tokenId": "ak-test"}) is None + assert ModalSandboxConfig.from_settings({"enabled": False, "tokenId": "ak-test", "tokenSecret": "as-test"}) is None + + +def test_modal_sandbox_config_parses_settings(): + config = ModalSandboxConfig.from_settings( + { + "enabled": True, + "tokenId": "ak-test", + "tokenSecret": "as-test", + "appName": "frameos-custom", + "image": "frameos/frameos:custom", + "timeout": "120", + "idleTimeout": "30", + "cpu": "4", + "memory": "8192", + "enableDocker": False, + } + ) + + assert isinstance(config, ModalSandboxConfig) + assert config.app_name == "frameos-custom" + assert config.image == "frameos/frameos:custom" + assert config.timeout == 120 + assert config.idle_timeout == 30 + assert config.cpu == 4 + assert config.memory == 8192 + assert config.enable_docker is False + + +def test_modal_sandbox_config_defaults_idle_timeout_to_60_seconds(): + config = ModalSandboxConfig.from_settings( + { + "enabled": True, + "tokenId": "ak-test", + "tokenSecret": "as-test", + } + ) + + assert isinstance(config, ModalSandboxConfig) + assert config.idle_timeout == 60 + + +def test_modal_sandbox_connection_summary_includes_resource_details(): + session = ModalSandboxSession( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + app_name="frameos-custom", + image="frameos/frameos:custom", + timeout=120, + idle_timeout=30, + cpu=4, + memory=8192, + region="us-east", + cloud="aws", + environment_name="prod", + enable_docker=False, + ) + ) + + summary = session._connection_summary({"host": "modal-host", "cpu_count": "8", "memory_mib": "16384"}) + + assert "Connected to Modal sandbox" in summary + assert "app=frameos-custom" in summary + assert "environment=prod" in summary + assert "host=modal-host" in summary + assert "image=frameos/frameos:custom" in summary + assert "cpu=4 requested" in summary + assert "memory=8192 MiB requested" in summary + assert "reported" not in summary + assert "region=us-east" in summary + assert "cloud=aws" in summary + assert "timeout=120s" in summary + assert "idle_timeout=30s" in summary + assert "nested_docker=disabled" in summary + + +def test_modal_sandbox_connection_summary_uses_requested_resource_labels_without_overrides(): + session = ModalSandboxSession( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + image="frameos/frameos:custom", + ) + ) + + summary = session._connection_summary({"host": "modal-host", "cpu_count": "8", "memory_mib": "16384"}) + + assert "cpu=auto requested" in summary + assert "memory=auto requested" in summary + assert "reported" not in summary + + +@pytest.mark.asyncio +async def test_modal_output_manager_logs_image_build_lines(): + entries = [] + + async def fake_log(level, message): + entries.append((level, message)) + + manager = _FrameOSModalOutputManager(SimpleNamespace(), fake_log) + + await manager.put_streaming_log(SimpleNamespace(file_descriptor=1, data="step 1\npartial")) + await manager.put_streaming_log(SimpleNamespace(file_descriptor=1, data=" done\n")) + await manager.put_streaming_log(SimpleNamespace(file_descriptor=2, data="error\n")) + await manager.flush_pending() + + assert entries == [ + ("stdout", "Modal image build: step 1"), + ("stdout", "Modal image build: partial done"), + ("stderr", "Modal image build: error"), + ] + assert manager.logged_lines == 3 + + +@pytest.mark.asyncio +async def test_modal_connect_logs_no_image_build_lines_notice(monkeypatch): + entries = [] + + async def fake_log(level, message): + entries.append((level, message)) + + class FakeOutputManager: + logged_lines = 0 + + async def flush_pending(self): + return None + + @contextlib.contextmanager + def fake_output_to_logs(_modal, _logger): + yield FakeOutputManager() + + class FakeClient: + @staticmethod + def from_credentials(_token_id, _token_secret): + return object() + + class FakeApp: + @staticmethod + def lookup(*_args, **_kwargs): + return object() + + class FakeImage: + @staticmethod + def from_registry(*_args, **_kwargs): + return object() + + class FakeSandbox: + @staticmethod + def create(*_args, **_kwargs): + raise RuntimeError("Image build for im-test failed. See build logs for more details.") + + monkeypatch.setitem( + sys.modules, + "modal", + SimpleNamespace(Client=FakeClient, App=FakeApp, Image=FakeImage, Sandbox=FakeSandbox), + ) + monkeypatch.setattr("app.utils.modal_sandbox._modal_output_to_logs", fake_output_to_logs) + + session = ModalSandboxSession( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + app_name="frameos-custom", + image="frameos/frameos:custom", + region="us-east", + cloud="aws", + ), + logger=fake_log, + ) + + with pytest.raises(RuntimeError): + await session._connect() + + assert entries == [ + ( + "stderr", + "Modal sandbox image build failed before Modal returned build-log lines. " + "app=frameos-custom, image=frameos/frameos:custom, region=us-east, cloud=aws. " + "Check the Modal dashboard for the image build record if the SDK error remains generic.", + ) + ] + + +def test_sandbox_sync_paths_for_docker_mounts(tmp_path): + src = tmp_path / "src" + cache = tmp_path / "cache" + src.mkdir() + cache.mkdir() + + command = f"docker run --rm -v {src}:/src -v {cache}:/cache image sh -lc 'echo ok'" + paths = sandbox_sync_paths_for_command(command) + + assert tmp_path in paths + + +def test_sandbox_sync_paths_preserves_command_path_spelling(tmp_path): + real_root = tmp_path / "private" / "var" / "folders" / "frameos-build" + real_root.mkdir(parents=True) + alias_root = tmp_path / "var" + alias_root.symlink_to(tmp_path / "private" / "var") + command_path = alias_root / "folders" / "frameos-build" + + command = f"cd {command_path} && nimble setup" + paths = sandbox_sync_paths_for_command(command) + + assert command_path in paths + assert real_root.resolve() not in paths + + +def test_parse_docker_run_command_extracts_modal_sandbox_shape(tmp_path): + work = tmp_path / "work" + cache = tmp_path / "cache" + command = ( + f"docker run --rm --platform linux/arm64 -v {work}:/work -v {cache}:/cache:ro " + "-e FORCE_UNSAFE_CONFIGURE=1 -w /work frameos/frameos-buildroot:latest bash /work/build.sh" + ) + + spec = parse_docker_run_command(command) + + assert spec is not None + assert spec.image == "frameos/frameos-buildroot:latest" + assert spec.platform == "linux/arm64" + assert spec.workdir == "/work" + assert spec.env == {"FORCE_UNSAFE_CONFIGURE": "1"} + assert [(mount.source, mount.target, mount.read_only) for mount in spec.mounts] == [ + (work, "/work", False), + (cache, "/cache", True), + ] + assert spec.args == ["bash", "/work/build.sh"] + + +@pytest.mark.asyncio +async def test_modal_sandbox_run_exports_frameos_path(monkeypatch): + calls = [] + + class FakeStream: + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + class FakeProcess: + stdout = FakeStream() + stderr = FakeStream() + + def wait(self): + return 0 + + class FakeSandbox: + def exec(self, *args, **kwargs): + calls.append((args, kwargs)) + return FakeProcess() + + session = ModalSandboxSession( + ModalSandboxConfig(enabled=True, token_id="ak-test", token_secret="as-test"), + ) + session._sandbox = FakeSandbox() + + status, _out, _err = await session.run("nimble --version", log_command=False) + + assert status == 0 + args, kwargs = calls[0] + assert args[:2] == ("bash", "-lc") + assert f"export PATH={FRAMEOS_SANDBOX_PATH}" in args[2] + assert args[2].endswith("; nimble --version") + assert kwargs["timeout"] == session.config.timeout + + +@pytest.mark.asyncio +async def test_modal_sandbox_run_logs_timeout_notice(): + entries = [] + + async def fake_log(level, message): + entries.append((level, message)) + + class FakeSandbox: + def exec(self, *_args, **_kwargs): + raise TimeoutError("sandbox command timed out") + + session = ModalSandboxSession( + ModalSandboxConfig( + enabled=True, + token_id="ak-test", + token_secret="as-test", + app_name="frameos-custom", + image="frameos/frameos:custom", + timeout=120, + idle_timeout=30, + ), + logger=fake_log, + ) + session._sandbox = FakeSandbox() + + with pytest.raises(TimeoutError): + await session.run("nimble --version", log_command=False, log_output=False) + + assert entries == [ + ( + "stderr", + "Modal sandbox timed out while running Modal command. " + "Configured timeout=120s, idle_timeout=30s, app=frameos-custom, " + "image=frameos/frameos:custom. Increase the Modal sandbox timeout/idle timeout " + "in global settings if this build legitimately needs longer.", + ) + ] diff --git a/backend/requirements.docker.in b/backend/requirements.docker.in index 4396ca876..3b79286cb 100644 --- a/backend/requirements.docker.in +++ b/backend/requirements.docker.in @@ -9,6 +9,7 @@ email_validator fastapi[standard] fonttools jwt +modal>=1.4.0 openai packaging pillow diff --git a/backend/requirements.in b/backend/requirements.in index 11553dab7..3941a7a60 100644 --- a/backend/requirements.in +++ b/backend/requirements.in @@ -8,6 +8,7 @@ fonttools honcho jwt mypy +modal>=1.4.0 packaging pillow pip-tools diff --git a/backend/requirements.txt b/backend/requirements.txt index 82e17557d..11336c190 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -6,6 +6,12 @@ # aiofiles==24.1.0 # via -r requirements.in +aiohappyeyeballs==2.6.2 + # via aiohttp +aiohttp==3.14.0 + # via modal +aiosignal==1.4.0 + # via aiohttp alembic==1.11.2 # via -r requirements.in annotated-types==0.7.0 @@ -20,14 +26,19 @@ arq==0.26.1 # via -r requirements.in asyncssh==2.19.0 # via -r requirements.in +attrs==26.1.0 + # via aiohttp backoff==2.2.1 # via posthog build==1.0.3 # via pip-tools +cbor2==6.1.2 + # via modal certifi==2023.7.22 # via # httpcore # httpx + # modal # requests cffi==1.15.1 # via cryptography @@ -38,6 +49,7 @@ charset-normalizer==3.2.0 click==8.1.7 # via # arq + # modal # pip-tools # rich-toolkit # typer @@ -70,14 +82,24 @@ filelock==3.13.1 # via virtualenv fonttools==4.55.3 # via -r requirements.in +frozenlist==1.8.0 + # via + # aiohttp + # aiosignal +grpclib==0.4.9 + # via modal h11==0.14.0 # via # httpcore # uvicorn +h2==4.3.0 + # via grpclib hiredis==3.1.0 # via redis honcho==1.1.0 # via -r requirements.in +hpack==4.1.0 + # via h2 httpcore==1.0.7 # via httpx httptools==0.6.4 @@ -86,6 +108,8 @@ httpx==0.28.1 # via # fastapi # openai +hyperframe==6.1.0 + # via h2 identify==2.5.33 # via pre-commit idna==3.4 @@ -94,6 +118,7 @@ idna==3.4 # email-validator # httpx # requests + # yarl iniconfig==2.0.0 # via pytest jinja2==3.1.2 @@ -113,6 +138,13 @@ markupsafe==2.1.3 # werkzeug mdurl==0.1.2 # via markdown-it-py +modal==1.4.3 + # via -r requirements.in +multidict==6.7.1 + # via + # aiohttp + # grpclib + # yarl mypy==1.13.0 # via -r requirements.in mypy-extensions==1.0.0 @@ -138,6 +170,12 @@ posthog==7.5.1 # via -r requirements.in pre-commit==3.6.0 # via -r requirements.in +propcache==0.5.2 + # via + # aiohttp + # yarl +protobuf==6.33.6 + # via modal pyasn1==0.6.1 # via # python-jose @@ -185,6 +223,7 @@ requests==2.31.0 # posthog rich==13.9.4 # via + # modal # rich-toolkit # typer rich-toolkit==0.12.0 @@ -213,10 +252,16 @@ sqlmodel==0.0.22 # via -r requirements.in starlette==0.41.3 # via fastapi +synchronicity==0.12.3 + # via modal +toml==0.10.2 + # via modal tqdm==4.67.1 # via openai typer==0.15.1 # via fastapi-cli +types-certifi==2021.10.8.3 + # via modal types-cffi==1.16.0.20240331 # via types-pyopenssl types-pillow==10.2.0.20240822 @@ -233,12 +278,17 @@ types-requests==2.32.0.20241016 # via -r requirements.in types-setuptools==75.6.0.20241126 # via types-cffi +types-toml==0.10.8.20260518 + # via modal typing-extensions==4.12.2 # via + # aiohttp + # aiosignal # alembic # anyio # asyncssh # fastapi + # modal # mypy # openai # posthog @@ -246,6 +296,7 @@ typing-extensions==4.12.2 # pydantic-core # rich-toolkit # sqlalchemy + # synchronicity # typer urllib3==2.0.4 # via @@ -260,13 +311,17 @@ uvloop==0.21.0 virtualenv==20.25.0 # via pre-commit watchfiles==1.0.3 - # via uvicorn + # via + # modal + # uvicorn websockets==14.1 # via uvicorn werkzeug==3.1.3 # via -r requirements.in wheel==0.41.3 # via pip-tools +yarl==1.24.2 + # via aiohttp # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/backend/tools/cross-toolchain.Dockerfile b/backend/tools/cross-toolchain.Dockerfile index 02446948f..75db38ea6 100644 --- a/backend/tools/cross-toolchain.Dockerfile +++ b/backend/tools/cross-toolchain.Dockerfile @@ -20,8 +20,48 @@ ARG TOOLCHAIN_PACKAGES="build-essential \ libjpeg-dev \ libfreetype6-dev \ libevdev-dev" +ARG TARGET_CROSS_DPKG_ARCHS="" +ARG TARGET_CROSS_PACKAGES="" RUN set -eu \ && apt-get update \ + && for arch in $TARGET_CROSS_DPKG_ARCHS; do \ + dpkg --add-architecture "$arch"; \ + done \ + && if [ -n "$TARGET_CROSS_DPKG_ARCHS" ] && grep -qi '^ID=ubuntu' /etc/os-release; then \ + native_arch="$(dpkg --print-architecture)"; \ + codename="$(. /etc/os-release && printf '%s' "${VERSION_CODENAME:-}")"; \ + if [ -z "$codename" ]; then \ + codename="$(awk -F= '/^UBUNTU_CODENAME=/{print $2}' /etc/os-release)"; \ + fi; \ + if [ -z "$codename" ]; then \ + echo "Unable to determine Ubuntu codename for cross-architecture apt sources" >&2; \ + exit 1; \ + fi; \ + foreign_archs="$TARGET_CROSS_DPKG_ARCHS"; \ + for source_file in /etc/apt/sources.list.d/ubuntu.sources; do \ + if [ -f "$source_file" ] && ! grep -q '^Architectures:' "$source_file"; then \ + awk -v arch="$native_arch" ' \ + /^Signed-By:/ { print; print "Architectures: " arch; next } \ + { print } \ + ' "$source_file" > "$source_file.tmp"; \ + mv "$source_file.tmp" "$source_file"; \ + fi; \ + done; \ + if [ -f /etc/apt/sources.list ]; then \ + sed -i -E "s#^deb (http://(archive|security)\\.ubuntu\\.com/ubuntu/)#deb [arch=${native_arch}] \\1#" /etc/apt/sources.list; \ + fi; \ + printf '%s\n' \ + 'Types: deb' \ + 'URIs: http://ports.ubuntu.com/ubuntu-ports/' \ + "Suites: ${codename} ${codename}-updates ${codename}-backports ${codename}-security" \ + 'Components: main universe restricted multiverse' \ + "Architectures: ${foreign_archs}" \ + 'Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg' \ + > /etc/apt/sources.list.d/ubuntu-ports.sources; \ + fi \ + && if [ -n "$TARGET_CROSS_DPKG_ARCHS" ]; then \ + apt-get update; \ + fi \ && SSL_PACKAGE="" \ && if apt-cache show libssl3t64 >/dev/null 2>&1; then \ SSL_PACKAGE="libssl3t64"; \ @@ -30,5 +70,5 @@ RUN set -eu \ elif apt-cache show libssl1.1 >/dev/null 2>&1; then \ SSL_PACKAGE="libssl1.1"; \ fi \ - && apt-get install -y --no-install-recommends $TOOLCHAIN_PACKAGES ${SSL_PACKAGE:+$SSL_PACKAGE} \ + && apt-get install -y --no-install-recommends $TOOLCHAIN_PACKAGES $TARGET_CROSS_PACKAGES ${SSL_PACKAGE:+$SSL_PACKAGE} \ && rm -rf /var/lib/apt/lists/* diff --git a/cross-toolchain-images.json b/cross-toolchain-images.json index 035989cb0..4804ead05 100644 --- a/cross-toolchain-images.json +++ b/cross-toolchain-images.json @@ -2,7 +2,7 @@ "images": { "frameos/frameos-cross-toolchain:debian_bookworm-linux_amd64-latest": { "base_image": "debian:bookworm", - "digest": "sha256:f9642dcfe712fdcdd96241f9e1574482e0b875fa9fd6098a29ac423a4cd82b59", + "digest": "sha256:6d58e95c6709d076b21b8d1829e0faa91523dcf3a69eb744315c290ee91a1136", "platform": "linux/amd64", "tag": "latest" }, @@ -20,7 +20,7 @@ }, "frameos/frameos-cross-toolchain:debian_trixie-linux_amd64-latest": { "base_image": "debian:trixie", - "digest": "sha256:89a8ff78ca6d028f47f4b893a2015f9f5a822751eee31a35573ae76d843844d9", + "digest": "sha256:dd72800e1fbef7c30922f010f284d2f23c0eb3d64c88d31b581b691f479e822a", "platform": "linux/amd64", "tag": "latest" }, @@ -38,7 +38,7 @@ }, "frameos/frameos-cross-toolchain:ubuntu_24.04-linux_amd64-latest": { "base_image": "ubuntu:24.04", - "digest": "sha256:181241f75de6ac7fcbea67d2f89c2f8cd40ed2675eeb0c14ce24d0ff1714e43c", + "digest": "sha256:384c54d83b82eaec1dc6ebcdbc838ab17b703f26436e997ccd4421e2392a0699", "platform": "linux/amd64", "tag": "latest" }, @@ -50,7 +50,7 @@ }, "frameos/frameos-cross-toolchain:ubuntu_26.04-linux_amd64-latest": { "base_image": "ubuntu:26.04", - "digest": "sha256:fbc0b8cd9f184672fcf5fb14ed38c153dda436a8b8cd73c17ac9ea18ebf4fc13", + "digest": "sha256:fbb5d7a736d4c66a87016c68c5a633411d9d955e40ea40a91566c757c0318ada", "platform": "linux/amd64", "tag": "latest" }, diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--full.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--full.png index 046ca9c19..4570e6805 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--full.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--full.png differ diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mid.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mid.png index be9b28853..aa820e935 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mid.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mid.png differ diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mobile.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mobile.png index 527a774c6..1f2c33ab1 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mobile.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--dark--mobile.png differ diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--full.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--full.png index e7aebe57a..b607a1490 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--full.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--full.png differ diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mid.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mid.png index a1a4c365b..1b07e5cbb 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mid.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mid.png differ diff --git a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mobile.png b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mobile.png index 195e525de..2cbe96e26 100644 Binary files a/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mobile.png and b/e2e/frontend-visual/snapshots/chromium/visual.spec.ts/global-settings--default--light--mobile.png differ diff --git a/e2e/frontend-visual/tests/frontend-e2e.spec.ts b/e2e/frontend-visual/tests/frontend-e2e.spec.ts index f1fe5e78c..64d558b1c 100644 --- a/e2e/frontend-visual/tests/frontend-e2e.spec.ts +++ b/e2e/frontend-visual/tests/frontend-e2e.spec.ts @@ -50,7 +50,7 @@ const globalSettingsSections = [ ['Home Assistant', 'settings-home-assistant'], ['GitHub', 'settings-github'], ['Unsplash API', 'settings-unsplash'], - ['Cross-compilation build host', 'settings-build-host'], + ['Build environment', 'settings-build-environment'], ['System information', 'settings-system'], ['Custom fonts', 'settings-fonts'], ] as const diff --git a/frontend/src/index.css b/frontend/src/index.css index 9458ba5e8..48a5225eb 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1597,7 +1597,58 @@ html[data-frameos-theme='dark'] .app-compiled-warning { } .settings-page-actions { + position: fixed; + top: 1.5rem; + right: 2rem; + z-index: 50; flex-wrap: wrap; + max-width: min(34rem, calc(100vw - var(--workspace-main-offset, 0px) - 3rem)); + padding: 0.5rem; + border: 1px solid var(--tool-border); + border-radius: 0.875rem; + background: var(--tool-bg-strong); + box-shadow: 0 18px 42px rgba(15, 23, 42, 0.14); + backdrop-filter: blur(16px); +} + +.frameos-app-shell .frameos-settings-nav-link-active { + background: var(--frameos-primary-soft); + color: var(--frameos-primary-text); +} + +.settings-nav-divider { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.settings-nav-divider::after { + content: ''; + min-width: 0; + flex: 1; + height: 1px; + background: color-mix(in srgb, currentColor 30%, transparent); +} + +.settings-group-divider { + display: flex; + align-items: center; + gap: 1rem; + margin: 2rem 0 0.75rem; + color: var(--frameos-muted-text); + font-size: 0.75rem; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.settings-group-divider::before, +.settings-group-divider::after { + content: ''; + min-width: 0; + flex: 1; + height: 1px; + background: color-mix(in srgb, currentColor 28%, transparent); } @media (min-width: 520px) { @@ -1612,6 +1663,36 @@ html[data-frameos-theme='dark'] .app-compiled-warning { } } +@media (min-width: 640px) and (max-width: 1023px) { + .settings-page-actions { + top: 4.8rem; + right: 1rem; + left: auto; + max-width: none; + justify-content: flex-end; + } +} + +@media (max-width: 639px) { + .settings-page-actions { + top: 4.8rem; + right: 0.75rem; + left: auto; + max-width: none; + justify-content: flex-end; + } +} + +@media (max-width: 499px) { + .settings-page-header { + padding-top: 3.75rem; + } + + .settings-page-actions { + top: 4.65rem; + } +} + .workspace-main-with-right-panel .terminal-command-bar { right: 520px; } diff --git a/frontend/src/models/framesModel.tsx b/frontend/src/models/framesModel.tsx index c8647087a..d76176ce8 100644 --- a/frontend/src/models/framesModel.tsx +++ b/frontend/src/models/framesModel.tsx @@ -95,6 +95,48 @@ function startBrowserDownload(path: string): void { } const pendingSdCardImageDownloads = new Set() +const sdCardImageProgressTimers = new Map>() +const SD_CARD_IMAGE_PROGRESS_INTERVAL_MS = 30 * 1000 + +function sdCardImageProgressDetail(startedAt: number): string { + const elapsedSeconds = Math.max(1, Math.round((Date.now() - startedAt) / 1000)) + const elapsedMinutes = Math.floor(elapsedSeconds / 60) + if (elapsedMinutes > 0) { + return `Still preparing SD card image (${elapsedMinutes} min elapsed)` + } + return 'Still preparing SD card image' +} + +function stopSdCardImageProgress(frameId: number): void { + const timer = sdCardImageProgressTimers.get(frameId) + if (timer === undefined || typeof window === 'undefined') { + return + } + window.clearInterval(timer) + sdCardImageProgressTimers.delete(frameId) +} + +function startSdCardImageProgress(frameId: number): void { + stopSdCardImageProgress(frameId) + if (typeof window === 'undefined') { + return + } + const startedAt = Date.now() + const updateProgressDetail = (): void => { + if (!pendingSdCardImageDownloads.has(frameId)) { + stopSdCardImageProgress(frameId) + return + } + longRunningTasksModel.actions.updateTaskProgress({ + frameId, + kind: 'buildrootImage', + progressCurrent: null, + progressTotal: null, + detail: sdCardImageProgressDetail(startedAt), + }) + } + sdCardImageProgressTimers.set(frameId, window.setInterval(updateProgressDetail, SD_CARD_IMAGE_PROGRESS_INTERVAL_MS)) +} export const framesModel = kea([ connect(() => ({ logic: [socketLogic, entityImagesModel] })), @@ -381,6 +423,7 @@ export const framesModel = kea([ const downloadUrl = sdImage?.downloadUrl || `/api/frames/${id}/buildroot/sd_image/download` pendingSdCardImageDownloads.add(id) + startSdCardImageProgress(id) longRunningTasksModel.actions.startTask({ frameId: id, kind: 'buildrootImage', @@ -401,6 +444,7 @@ export const framesModel = kea([ const data = await response.json() if (data?.sdImage?.status === 'ready') { pendingSdCardImageDownloads.delete(id) + stopSdCardImageProgress(id) startBrowserDownload(data.sdImage.downloadUrl || downloadUrl) longRunningTasksModel.actions.finishTask({ frameId: id, @@ -412,6 +456,7 @@ export const framesModel = kea([ } } catch (error) { pendingSdCardImageDownloads.delete(id) + stopSdCardImageProgress(id) longRunningTasksModel.actions.taskFailed({ frameId: id, kind: 'buildrootImage', @@ -427,9 +472,11 @@ export const framesModel = kea([ } if (sdImage.status === 'ready') { pendingSdCardImageDownloads.delete(frame.id) + stopSdCardImageProgress(frame.id) startBrowserDownload(sdImage.downloadUrl || `/api/frames/${frame.id}/buildroot/sd_image/download`) } else if (sdImage.status === 'error' || sdImage.status === 'missing' || sdImage.status === 'stale') { pendingSdCardImageDownloads.delete(frame.id) + stopSdCardImageProgress(frame.id) } }, setDeployWithAgent: async ({ id, deployWithAgent }) => { diff --git a/frontend/src/models/longRunningTasksModel.ts b/frontend/src/models/longRunningTasksModel.ts index 8dc818169..33c6b478a 100644 --- a/frontend/src/models/longRunningTasksModel.ts +++ b/frontend/src/models/longRunningTasksModel.ts @@ -63,8 +63,8 @@ export interface UpdateTaskProgressPayload { frameId: number kind?: LongRunningTaskKind sceneId?: string | null - progressCurrent: number - progressTotal: number + progressCurrent: number | null + progressTotal: number | null detail?: string | null } diff --git a/frontend/src/scenes/frame/frameDeployUtils.ts b/frontend/src/scenes/frame/frameDeployUtils.ts index 671b35dde..a3657387d 100644 --- a/frontend/src/scenes/frame/frameDeployUtils.ts +++ b/frontend/src/scenes/frame/frameDeployUtils.ts @@ -38,6 +38,7 @@ export interface FullDeployPlanResponse { will_attempt_precompiled?: boolean cross_compile_supported?: boolean build_host_configured?: boolean + build_executor?: string | null prebuilt_target?: string | null has_prebuilt_entry?: boolean precompiled_release_url?: string | null @@ -270,7 +271,7 @@ function inferBuildStrategy(frame?: Partial | null): string { ? 'Build on device' : crossCompilation === 'always' ? 'Cross-compile' - : 'Cross-compile if the detected target supports it, otherwise build on device' + : 'Use the global build environment' if (compilationMode === 'precompiled') { if (crossCompilation === 'always') { @@ -428,8 +429,8 @@ export function buildFullDeployPlanSummary( value: fullPlan.binary.will_attempt_precompiled ? 'Download and install the precompiled FrameOS release' : fullPlan.binary.will_attempt_cross_compile - ? fullPlan.binary.build_host_configured - ? 'Cross-compile on the configured build host' + ? fullPlan.binary.build_executor + ? `Cross-compile via ${fullPlan.binary.build_executor}` : 'Cross-compile locally on this server' : fullPlan.binary.cross_compile_supported ? 'Build on device because cross-compilation is disabled' diff --git a/frontend/src/scenes/frame/panels/FrameSettings/FrameSettings.tsx b/frontend/src/scenes/frame/panels/FrameSettings/FrameSettings.tsx index e0ae7b529..71258610d 100644 --- a/frontend/src/scenes/frame/panels/FrameSettings/FrameSettings.tsx +++ b/frontend/src/scenes/frame/panels/FrameSettings/FrameSettings.tsx @@ -679,14 +679,11 @@ export function FrameSettings({ tooltip={

- Choose how to build the FrameOS binary: auto will try to cross-compile and fall back to on-device - builds, always will fail if cross compilation is unavailable, and never will always build on the - device. -

-

- If you're running FrameOS via Docker, you may need to configure a build host for cross-compilation - on the global settings page. + Choose how to build the FrameOS binary: auto follows the build environment selected in global + settings, always fails if server-side compilation is disabled or unavailable, and never always + builds on the device.

+

Configure Docker, a build host, or Modal sandboxes from global settings.

} > diff --git a/frontend/src/scenes/settings/Settings.tsx b/frontend/src/scenes/settings/Settings.tsx index 03e33dfc1..0a7897039 100644 --- a/frontend/src/scenes/settings/Settings.tsx +++ b/frontend/src/scenes/settings/Settings.tsx @@ -1,5 +1,7 @@ import { useActions, useValues } from 'kea' import { Form, Group } from 'kea-forms' +import { useEffect, useRef, useState, type MouseEvent as ReactMouseEvent } from 'react' +import clsx from 'clsx' import { Box } from '../../components/Box' import { settingsLogic } from './settingsLogic' import { Spinner } from '../../components/Spinner' @@ -26,22 +28,48 @@ import { isMobileWorkspaceViewport, workspaceLogic } from '../workspace/workspac import { accountLogic } from './accountLogic' import versions from '../../../../versions.json' import { timezoneOptions } from '../../decorators/timezones' +import { systemInfoLogic } from './systemInfoLogic' -const settingsNavItems = [ - ['Account', '#settings-account'], - ['Defaults', '#settings-defaults'], - ['SSH Keys', '#settings-ssh'], - ['FrameOS Gallery', '#settings-gallery'], - ['OpenAI', '#settings-openai'], - ['PostHog', '#settings-posthog'], - ['Home Assistant', '#settings-home-assistant'], - ['GitHub', '#settings-github'], - ['Unsplash API', '#settings-unsplash'], - ['Cross-compilation build host', '#settings-build-host'], - ['System information', '#settings-system'], - ['Custom fonts', '#settings-fonts'], +type SettingsNavItem = readonly [string, string] +type SettingsNavSection = { + label: string + items: readonly SettingsNavItem[] +} +type SettingsSectionId = string + +const settingsNavSections: readonly SettingsNavSection[] = [ + { + label: '', + items: [['Account', '#settings-account']], + }, + { + label: 'Settings', + items: [ + ['Defaults', '#settings-defaults'], + ['SSH Keys', '#settings-ssh'], + ['Build environment', '#settings-build-environment'], + ['Custom fonts', '#settings-fonts'], + ], + }, + { + label: 'Services', + items: [ + ['FrameOS Gallery', '#settings-gallery'], + ['OpenAI', '#settings-openai'], + ['PostHog', '#settings-posthog'], + ['Home Assistant', '#settings-home-assistant'], + ['GitHub', '#settings-github'], + ['Unsplash API', '#settings-unsplash'], + ], + }, + { + label: 'Information', + items: [['System information', '#settings-system']], + }, ] as const +const settingsNavItems = settingsNavSections.flatMap((section) => section.items) + const frameosVersion = typeof versions.frameos === 'string' ? versions.frameos.split('+')[0] : null const frameosVersionLabel = frameosVersion ? `FrameOS ${frameosVersion}` : 'FrameOS' @@ -61,12 +89,8 @@ function scrollToSettingsSection(sectionId: string, attempt = 0): void { window.requestAnimationFrame(() => { const section = document.getElementById(sectionId) if (section) { - if (isMobileWorkspaceViewport()) { - const top = section.getBoundingClientRect().top + window.scrollY - settingsHeaderOffset() - window.scrollTo({ top: Math.max(0, top), behavior: 'smooth' }) - } else { - section.scrollIntoView({ behavior: 'smooth', block: 'start' }) - } + const top = section.getBoundingClientRect().top + window.scrollY - settingsHeaderOffset() + window.scrollTo({ top: Math.max(0, top), behavior: 'smooth' }) return } @@ -76,7 +100,38 @@ function scrollToSettingsSection(sectionId: string, attempt = 0): void { }) } -function AccountSettingsSection(): JSX.Element { +function activeSettingsSectionId(): SettingsSectionId { + const viewportTop = settingsHeaderOffset() + 8 + const viewportBottom = typeof window === 'undefined' ? viewportTop : window.innerHeight + const candidates = settingsNavItems + .map(([_label, href]) => { + const section = document.getElementById(href.slice(1)) + if (!section) { + return null + } + const rect = section.getBoundingClientRect() + return { href, rect } + }) + .filter((candidate): candidate is { href: SettingsSectionId; rect: DOMRect } => { + return !!candidate && candidate.rect.bottom >= viewportTop && candidate.rect.top <= viewportBottom + }) + + if (candidates.length === 0) { + return settingsNavItems[0][1] + } + + const current = + candidates + .filter((candidate) => candidate.rect.top <= viewportTop) + .toSorted((first, second) => second.rect.top - first.rect.top)[0] ?? + candidates.toSorted( + (first, second) => Math.abs(first.rect.top - viewportTop) - Math.abs(second.rect.top - viewportTop) + )[0] + + return current.href +} + +function AccountSettingsSection({ onLogout }: { onLogout: () => void }): JSX.Element { const { account, accountEmail, @@ -96,12 +151,15 @@ function AccountSettingsSection(): JSX.Element { return (
-
- Account -
+
+
Account
+ +
-
-
+
+
{emailEditorOpen ? ( @@ -190,8 +248,8 @@ function AccountSettingsSection(): JSX.Element {
) : ( -
-
+
+
@@ -224,6 +282,14 @@ function IngressAccountSettingsSection(): JSX.Element { ) } +function SettingsGroupDivider({ label }: { label: string }): JSX.Element { + return ( + + ) +} + export function Settings() { const { settings, @@ -237,6 +303,8 @@ export function Settings() { customFonts, generatingSshKeyId, sshKeyExpandedIds, + isTestingBuildHost, + isTestingModalSandbox, } = useValues(settingsLogic) const { framesList } = useValues(framesModel) const { @@ -246,49 +314,106 @@ export function Settings() { generateSshKey, removeSshKey, newBuildHostKey, + testBuildHost, + testModalSandbox, deleteCustomFont, setSettingsValue, toggleSshKeyExpanded, toggleOpenAiModelOverrides, } = useActions(settingsLogic) + const { systemInfo } = useValues(systemInfoLogic) + const { loadSystemInfo } = useActions(systemInfoLogic) const { isHassioIngress } = useValues(sceneLogic) const { logout } = useActions(sceneLogic) const { closeSecondarySidebar } = useActions(workspaceLogic) const defaultSshKeyIds = getDefaultSshKeyIds(settings?.ssh_keys) + const buildEnvironmentProvider = settings?.buildEnvironment?.provider || 'docker' + const [activeSettingsSection, setActiveSettingsSection] = useState(settingsNavItems[0][1]) + const settingsNavLinkRefs = useRef>({}) const framesUsingKey = (keyId: string) => framesList.filter((frame) => (frame.ssh_keys ?? defaultSshKeyIds).includes(keyId)) - const settingsTree = ( -
- {settingsNavItems.map(([label, href]) => ( - { - if (!isMobileWorkspaceViewport()) { - return - } - event.preventDefault() - closeSecondarySidebar() - window.history.pushState(null, '', `${window.location.pathname}${window.location.search}${href}`) - scrollToSettingsSection(href.slice(1)) - }} - className="frameos-settings-nav-link block rounded-xl px-3 py-2.5 text-base font-medium text-slate-700 transition hover:bg-slate-100 focus:outline-none focus-visible:ring-2 focus-visible:ring-blue-400" - > - {label} - + useEffect(() => { + if (typeof window === 'undefined') { + return + } + + let frame: number | null = null + const scheduleUpdate = (): void => { + if (frame !== null) { + return + } + frame = window.requestAnimationFrame(() => { + frame = null + setActiveSettingsSection(activeSettingsSectionId()) + }) + } + + scheduleUpdate() + window.addEventListener('scroll', scheduleUpdate, { passive: true }) + window.addEventListener('resize', scheduleUpdate) + return () => { + if (frame !== null) { + window.cancelAnimationFrame(frame) + } + window.removeEventListener('scroll', scheduleUpdate) + window.removeEventListener('resize', scheduleUpdate) + } + }, [buildEnvironmentProvider, customFonts.length, openAiModelOverridesExpanded, savedSettingsLoading]) + + useEffect(() => { + settingsNavLinkRefs.current[activeSettingsSection]?.scrollIntoView({ block: 'nearest' }) + }, [activeSettingsSection]) + + const handleSettingsNavClick = (event: ReactMouseEvent, href: SettingsSectionId): void => { + if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey || event.altKey) { + return + } + + event.preventDefault() + setActiveSettingsSection(href) + if (isMobileWorkspaceViewport()) { + closeSecondarySidebar() + } + window.history.pushState(null, '', `${window.location.pathname}${window.location.search}${href}`) + scrollToSettingsSection(href.slice(1)) + } + + const settingsTree = ( + ) const settingsActions = (
-
{frameosVersionLabel}
- {!isHassioIngress ? ( - - ) : null}