diff --git a/pyproject.toml b/pyproject.toml index a24e799..cce12b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "pydantic>=2,<3", "Jinja2", "pyyaml", - "docker", + "python-on-whales", "python-gitlab", "cachetools", "blinker", diff --git a/shard_core/service/app_installation/__init__.py b/shard_core/service/app_installation/__init__.py index bf32aef..79158d8 100644 --- a/shard_core/service/app_installation/__init__.py +++ b/shard_core/service/app_installation/__init__.py @@ -1,3 +1,4 @@ +import asyncio import logging from shard_core.database import database @@ -6,7 +7,8 @@ from shard_core.data_model.app_meta import InstallationReason, InstalledApp, Status from shard_core.util import signals from shard_core.settings import settings -from shard_core.util.subprocess import subprocess, SubprocessError +from python_on_whales import DockerClient +from python_on_whales.exceptions import DockerException from . import util, worker from .exceptions import AppAlreadyInstalled, AppDoesNotExist, AppNotInstalled @@ -122,12 +124,11 @@ async def refresh_init_apps(): async def login_docker_registries(): registries = settings().apps.registries + client = DockerClient() for r in registries: try: - await subprocess( - "docker", "login", "-u", r.username, "-p", r.password, r.uri - ) - except (SubprocessError, OSError) as e: + await asyncio.to_thread(client.login, server=r.uri, username=r.username, password=r.password) + except (DockerException, OSError) as e: log.error(f"could not log in to registry {r.uri}: {e}") else: log.debug(f"logged in to registry {r.uri}") diff --git a/shard_core/service/app_tools.py b/shard_core/service/app_tools.py index 6d0c165..e84f765 100644 --- a/shard_core/service/app_tools.py +++ b/shard_core/service/app_tools.py @@ -3,6 +3,9 @@ import logging from pathlib import Path +from python_on_whales import DockerClient +from python_on_whales.exceptions import DockerException + import shard_core.data_model.profile from shard_core.database.connection import db_conn from shard_core.database import installed_apps as db_installed_apps @@ -15,16 +18,14 @@ from shard_core.settings import settings from shard_core.util import signals from shard_core.util.misc import throttle -from shard_core.util.subprocess import subprocess, SubprocessError log = logging.getLogger(__name__) async def docker_create_app_containers(name: str): log.debug(f"creating containers for app {name}") - await subprocess( - "docker-compose", "up", "--no-start", cwd=get_installed_apps_path() / name - ) + client = DockerClient(compose_project_directory=get_installed_apps_path() / name) + await asyncio.to_thread(client.compose.up, start=False) @throttle(5) @@ -35,31 +36,22 @@ async def docker_start_app(name: str): if app_status in [Status.STOPPED, Status.RUNNING, Status.DOWN]: log.debug(f"starting app {name=}") + client = DockerClient(compose_project_directory=get_installed_apps_path() / name) try: - await subprocess( - "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name - ) - except SubprocessError as e: + await asyncio.to_thread(client.compose.up, detach=True) + except DockerException as e: if "network" in str(e) and "not found" in str(e): log.warning( f"stale network reference for app {name=}, recreating containers" ) - await subprocess( - "docker-compose", "down", cwd=get_installed_apps_path() / name - ) - await subprocess( - "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name - ) + await asyncio.to_thread(client.compose.down) + await asyncio.to_thread(client.compose.up, detach=True) elif "Conflict" in str(e) and "already in use" in str(e): log.warning( f"stale containers for app {name=}, removing and recreating" ) - await subprocess( - "docker-compose", "down", cwd=get_installed_apps_path() / name - ) - await subprocess( - "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name - ) + await asyncio.to_thread(client.compose.down) + await asyncio.to_thread(client.compose.up, detach=True) else: raise async with db_conn() as conn: @@ -74,7 +66,8 @@ async def docker_stop_app(name: str, set_status: bool = True): app = await db_installed_apps.get_by_name(conn, name) app_status = app["status"] if app else None if app_status in [Status.RUNNING, Status.UNINSTALLING]: - await subprocess("docker-compose", "stop", cwd=get_installed_apps_path() / name) + client = DockerClient(compose_project_directory=get_installed_apps_path() / name) + await asyncio.to_thread(client.compose.stop) if set_status: async with db_conn() as conn: await db_installed_apps.update_status(conn, name, Status.STOPPED) @@ -88,7 +81,8 @@ async def docker_shutdown_app(name: str, set_status: bool = True, force: bool = app = await db_installed_apps.get_by_name(conn, name) app_status = app["status"] if app else None if force or app_status in [Status.STOPPED, Status.UNINSTALLING]: - await subprocess("docker-compose", "down", cwd=get_installed_apps_path() / name) + client = DockerClient(compose_project_directory=get_installed_apps_path() / name) + await asyncio.to_thread(client.compose.down) if set_status: async with db_conn() as conn: await db_installed_apps.update_status(conn, name, Status.DOWN) @@ -154,15 +148,15 @@ def enrich_installed_app_with_meta(installed_app: InstalledApp) -> InstalledAppW async def docker_prune_images(apply_filter=True): - command = ["docker", "image", "prune", "-fa"] - if apply_filter: - command.extend(["--filter", f"until={settings().apps.pruning.max_age}h"]) + filters = {"until": f"{settings().apps.pruning.max_age}h"} if apply_filter else {} try: - stdout = await subprocess(*command) - except SubprocessError as e: + output = await asyncio.to_thread( + DockerClient().image.prune, all=True, filters=filters + ) + except DockerException as e: log.error(f"failed to prune docker images: {e}") return - lines = stdout.splitlines() + lines = output.splitlines() log.info(f"docker images pruned, {lines[-1]}") return lines[-1] diff --git a/shard_core/web/internal/app_error.py b/shard_core/web/internal/app_error.py index 053c2a4..50faed2 100644 --- a/shard_core/web/internal/app_error.py +++ b/shard_core/web/internal/app_error.py @@ -3,9 +3,9 @@ from functools import lru_cache from pathlib import Path -import docker import jinja2 -from docker import errors as docker_errors +from python_on_whales import DockerClient +from python_on_whales.exceptions import NoSuchContainer from fastapi import APIRouter, Request from fastapi.responses import HTMLResponse from pydantic import BaseModel @@ -79,8 +79,8 @@ async def get_splash_behaviour(request: Request): def get_container_status(app_name): docker_client = get_docker_client() try: - status = docker_client.containers.get(app_name).status - except docker_errors.NotFound: + status = docker_client.container.inspect(app_name).state.status + except NoSuchContainer: status = "unknown" return status @@ -93,7 +93,7 @@ def get_app_name(request): @lru_cache() def get_docker_client(): - return docker.from_env() + return DockerClient() @lru_cache() diff --git a/shard_core/web/internal/call_peer.py b/shard_core/web/internal/call_peer.py index 0d5f087..59dae1b 100644 --- a/shard_core/web/internal/call_peer.py +++ b/shard_core/web/internal/call_peer.py @@ -2,8 +2,7 @@ from functools import lru_cache from typing import List -import docker -from docker.models.containers import Container +from python_on_whales import DockerClient, Container from fastapi import APIRouter, Request from starlette.responses import StreamingResponse @@ -33,14 +32,16 @@ async def call_peer(id_: str, rest: str, request: Request): def _get_app_for_ip_address(ip_address: str): # todo: test this docker_client = _get_docker_client() - docker_client.networks.get("portal") - containers: List[Container] = docker_client.containers.list() + docker_client.network.inspect("portal") + containers: List[Container] = docker_client.container.list() for c in containers: - if c.attrs["NetworkSettings"]["Networks"]["portal"]["IPAddress"] == ip_address: + networks = c.network_settings.networks or {} + portal_net = networks.get("portal") + if portal_net and portal_net.ip_address == ip_address: return c.name raise RuntimeError(f"no running container found for address {ip_address}") @lru_cache() def _get_docker_client(): - return docker.from_env() + return DockerClient() diff --git a/tests/test_app_installation.py b/tests/test_app_installation.py index 6f347f4..ec843f2 100644 --- a/tests/test_app_installation.py +++ b/tests/test_app_installation.py @@ -1,6 +1,6 @@ -import docker import pytest -from docker.errors import NotFound +from python_on_whales import DockerClient +from python_on_whales.exceptions import NoSuchContainer from fastapi import status from httpx import AsyncClient @@ -21,7 +21,7 @@ async def test_get_initial_apps(api_client: AsyncClient): async def test_install_app(api_client: AsyncClient): - docker_client = docker.from_env() + docker_client = DockerClient() app_name = "mock_app" response = await api_client.post(f"protected/apps/{app_name}") @@ -29,7 +29,7 @@ async def test_install_app(api_client: AsyncClient): await wait_until_app_installed(api_client, app_name) - docker_client.containers.get(app_name) + docker_client.container.inspect(app_name) response = (await api_client.get("protected/apps")).json() assert len(response) == 4 @@ -66,8 +66,8 @@ async def test_install_app_twice(api_client: AsyncClient): async def test_uninstall_app(api_client: AsyncClient): - docker_client = docker.from_env() - docker_client.containers.get("filebrowser") + docker_client = DockerClient() + docker_client.container.inspect("filebrowser") response = await api_client.delete("protected/apps/filebrowser") assert response.status_code == status.HTTP_204_NO_CONTENT @@ -77,12 +77,12 @@ async def test_uninstall_app(api_client: AsyncClient): response = (await api_client.get("protected/apps")).json() assert len(response) == 2 - with pytest.raises(NotFound): - docker_client.containers.get("filebrowser") + with pytest.raises(NoSuchContainer): + docker_client.container.inspect("filebrowser") async def test_uninstall_running_app(api_client: AsyncClient): - docker_client = docker.from_env() + docker_client = DockerClient() app_name = "mock_app" response = await api_client.post(f"protected/apps/{app_name}") @@ -108,13 +108,13 @@ async def test_uninstall_running_app(api_client: AsyncClient): response = (await api_client.get("protected/apps")).json() assert len(response) == 3 # Initial apps are still installed - with pytest.raises(NotFound): - docker_client.containers.get(app_name) + with pytest.raises(NoSuchContainer): + docker_client.container.inspect(app_name) async def test_install_custom_app(api_client: AsyncClient): app_name = "mock_app" - docker_client = docker.from_env() + docker_client = DockerClient() app_zip = mock_app_store_path() / app_name / f"{app_name}.zip" with open(app_zip, "rb") as f: @@ -127,7 +127,7 @@ async def test_install_custom_app(api_client: AsyncClient): await wait_until_app_installed(api_client, app_name) - docker_client.containers.get(app_name) + docker_client.container.inspect(app_name) response = (await api_client.get("protected/apps")).json() assert len(response) == 4 diff --git a/tests/test_app_lifecycle.py b/tests/test_app_lifecycle.py index 45a7a92..6ca80f7 100644 --- a/tests/test_app_lifecycle.py +++ b/tests/test_app_lifecycle.py @@ -1,4 +1,4 @@ -import docker +from python_on_whales import DockerClient from fastapi import status from shard_core.data_model.app_meta import InstalledApp, Status @@ -6,7 +6,7 @@ async def test_app_starts_and_stops(requests_mock, api_client): - docker_client = docker.from_env() + docker_client = DockerClient() app_name = "quick_stop" response = await api_client.post(f"protected/apps/{app_name}") @@ -14,7 +14,7 @@ async def test_app_starts_and_stops(requests_mock, api_client): await wait_until_app_installed(api_client, app_name) - assert docker_client.containers.get(app_name).status == "created" + assert docker_client.container.inspect(app_name).state.status == "created" assert ( InstalledApp.model_validate( (await api_client.get(f"protected/apps/{app_name}")).json() @@ -32,7 +32,7 @@ async def test_app_starts_and_stops(requests_mock, api_client): response.raise_for_status() async def assert_app_running(): - assert docker_client.containers.get(app_name).status == "running" + assert docker_client.container.inspect(app_name).state.status == "running" assert ( InstalledApp.model_validate( (await api_client.get(f"protected/apps/{app_name}")).json() @@ -41,7 +41,7 @@ async def assert_app_running(): ) async def assert_app_exited(): - assert docker_client.containers.get(app_name).status == "exited" + assert docker_client.container.inspect(app_name).state.status == "exited" assert ( InstalledApp.model_validate( (await api_client.get(f"protected/apps/{app_name}")).json() @@ -78,7 +78,7 @@ async def assert_app_exited(): async def test_always_on_app_starts(requests_mock, api_client): - docker_client = docker.from_env() + docker_client = DockerClient() app_name = "always_on" response = await api_client.post(f"protected/apps/{app_name}") @@ -87,7 +87,7 @@ async def test_always_on_app_starts(requests_mock, api_client): await wait_until_app_installed(api_client, app_name) async def assert_app_running(): - assert docker_client.containers.get(app_name).status == "running" + assert docker_client.container.inspect(app_name).state.status == "running" assert ( InstalledApp.model_validate( (await api_client.get(f"protected/apps/{app_name}")).json() diff --git a/tests/util.py b/tests/util.py index 358b7b9..a5c441e 100644 --- a/tests/util.py +++ b/tests/util.py @@ -15,7 +15,8 @@ from requests_http_signature import HTTPSignatureAuth from shard_core.data_model.app_meta import InstalledApp, Status -from shard_core.util.subprocess import subprocess, SubprocessError +from python_on_whales import DockerClient +from python_on_whales.exceptions import DockerException WAITING_DOCKER_IMAGE = "nginx:alpine" @@ -137,9 +138,10 @@ def modify_request_like_traefik_forward_auth(request: PreparedRequest) -> Reques @asynccontextmanager async def docker_network_portal(): + docker_client = DockerClient() try: - await subprocess("docker", "network", "create", "portal") - except SubprocessError: + await asyncio.to_thread(docker_client.network.create, "portal") + except DockerException: pass # network already exists try: yield @@ -149,26 +151,19 @@ async def docker_network_portal(): async def _force_remove_docker_network(network: str): """Disconnect all containers from a network, then remove it.""" + docker_client = DockerClient() try: - output = await subprocess( - "docker", - "network", - "inspect", - network, - "--format", - "{{range .Containers}}{{.Name}} {{end}}", - ) - container_names = output.split() - for name in container_names: + net = await asyncio.to_thread(docker_client.network.inspect, network) + for container in (net.containers or {}).values(): try: - await subprocess("docker", "network", "disconnect", "-f", network, name) - except SubprocessError: + await asyncio.to_thread(docker_client.network.disconnect, network, container.name, force=True) + except DockerException: pass - except SubprocessError: + except DockerException: pass # network doesn't exist, nothing to do try: - await subprocess("docker", "network", "rm", network) - except SubprocessError: + await asyncio.to_thread(docker_client.network.remove, network) + except DockerException: pass diff --git a/uv.lock b/uv.lock index a94159d..357d1af 100644 --- a/uv.lock +++ b/uv.lock @@ -549,20 +549,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - [[package]] name = "email-validator" version = "2.3.0" @@ -1507,6 +1493,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/a2/43bbc5860b5034e2af4ef99a0e04d726ff329c43e192ef3abaa8d7ecfce5/python_multipart-0.0.28-py3-none-any.whl", hash = "sha256:10faac07eb966c3f48dc415f9dee46c04cb10d58d30a35677db8027c825ed9b6", size = 29438, upload-time = "2026-05-10T11:05:15.052Z" }, ] +[[package]] +name = "python-on-whales" +version = "0.81.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/81/4b545de811e41ff90762a1c7045fcae215f20f4114eebddedab8eab6f63e/python_on_whales-0.81.0.tar.gz", hash = "sha256:bb6172014e3fe949f908092748bddf34df758cb92c2c3d0536b75bf236abce1b", size = 115066, upload-time = "2026-03-09T14:17:43.534Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/cb/5049e041a1d0e6f6cb6ac8737a8215fb4e67cb147f140ef31e67361dc61a/python_on_whales-0.81.0-py3-none-any.whl", hash = "sha256:6d5f81f56d0f95fd311a7cce29a01a1a60841074e4936000dc5f2dc9a7ffafac", size = 119240, upload-time = "2026-03-09T14:17:42.195Z" }, +] + [[package]] name = "pytokens" version = "0.4.1" @@ -1531,19 +1530,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, ] -[[package]] -name = "pywin32" -version = "311" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, - { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, - { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, - { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714, upload-time = "2025-07-14T20:13:32.449Z" }, - { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800, upload-time = "2025-07-14T20:13:34.312Z" }, - { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540, upload-time = "2025-07-14T20:13:36.379Z" }, -] - [[package]] name = "pyyaml" version = "6.0.3" @@ -1777,7 +1763,6 @@ dependencies = [ { name = "cachetools" }, { name = "croniter" }, { name = "cryptography" }, - { name = "docker" }, { name = "email-validator" }, { name = "fastapi", extra = ["standard"] }, { name = "httpx" }, @@ -1789,6 +1774,7 @@ dependencies = [ { name = "pyjwt" }, { name = "python-gitlab" }, { name = "python-multipart" }, + { name = "python-on-whales" }, { name = "pyyaml" }, { name = "requests" }, { name = "requests-http-signature" }, @@ -1825,7 +1811,6 @@ requires-dist = [ { name = "croniter" }, { name = "cryptography" }, { name = "datamodel-code-generator", extras = ["http"], marker = "extra == 'dev'" }, - { name = "docker" }, { name = "email-validator" }, { name = "fastapi", extras = ["standard"] }, { name = "httpx" }, @@ -1841,6 +1826,7 @@ requires-dist = [ { name = "pytest-mock", marker = "extra == 'dev'" }, { name = "python-gitlab" }, { name = "python-multipart" }, + { name = "python-on-whales" }, { name = "pyyaml" }, { name = "requests" }, { name = "requests-http-signature" },