Fix #24: Integrate python-on-whales for Docker operations

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
     "pydantic>=2,<3",
     "Jinja2",
     "pyyaml",
-    "docker",
+    "python-on-whales",
     "python-gitlab",
     "cachetools",
     "blinker",

shard_core/service/app_installation/__init__.py

@@ -1,3 +1,4 @@
+import asyncio
 import logging
 
 from shard_core.database import database
@@ -6,7 +7,8 @@
 from shard_core.data_model.app_meta import InstallationReason, InstalledApp, Status
 from shard_core.util import signals
 from shard_core.settings import settings
-from shard_core.util.subprocess import subprocess, SubprocessError
+from python_on_whales import DockerClient
+from python_on_whales.exceptions import DockerException
 from . import util, worker
 from .exceptions import AppAlreadyInstalled, AppDoesNotExist, AppNotInstalled
 
@@ -122,12 +124,11 @@ async def refresh_init_apps():
 
 async def login_docker_registries():
     registries = settings().apps.registries
+    client = DockerClient()
     for r in registries:
         try:
-            await subprocess(
-                "docker", "login", "-u", r.username, "-p", r.password, r.uri
-            )
-        except (SubprocessError, OSError) as e:
+            await asyncio.to_thread(client.login, server=r.uri, username=r.username, password=r.password)
+        except (DockerException, OSError) as e:
             log.error(f"could not log in to registry {r.uri}: {e}")
         else:
             log.debug(f"logged in to registry {r.uri}")

shard_core/service/app_tools.py

@@ -3,6 +3,9 @@
 import logging
 from pathlib import Path
 
+from python_on_whales import DockerClient
+from python_on_whales.exceptions import DockerException
+
 import shard_core.data_model.profile
 from shard_core.database.connection import db_conn
 from shard_core.database import installed_apps as db_installed_apps
@@ -15,16 +18,14 @@
 from shard_core.settings import settings
 from shard_core.util import signals
 from shard_core.util.misc import throttle
-from shard_core.util.subprocess import subprocess, SubprocessError
 
 log = logging.getLogger(__name__)
 
 
 async def docker_create_app_containers(name: str):
     log.debug(f"creating containers for app {name}")
-    await subprocess(
-        "docker-compose", "up", "--no-start", cwd=get_installed_apps_path() / name
-    )
+    client = DockerClient(compose_project_directory=get_installed_apps_path() / name)
+    await asyncio.to_thread(client.compose.up, start=False)
 
 
 @throttle(5)
@@ -35,31 +36,22 @@ async def docker_start_app(name: str):
 
     if app_status in [Status.STOPPED, Status.RUNNING, Status.DOWN]:
         log.debug(f"starting app {name=}")
+        client = DockerClient(compose_project_directory=get_installed_apps_path() / name)
         try:
-            await subprocess(
-                "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name
-            )
-        except SubprocessError as e:
+            await asyncio.to_thread(client.compose.up, detach=True)
+        except DockerException as e:
             if "network" in str(e) and "not found" in str(e):
                 log.warning(
                     f"stale network reference for app {name=}, recreating containers"
                 )
-                await subprocess(
-                    "docker-compose", "down", cwd=get_installed_apps_path() / name
-                )
-                await subprocess(
-                    "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name
-                )
+                await asyncio.to_thread(client.compose.down)
+                await asyncio.to_thread(client.compose.up, detach=True)
             elif "Conflict" in str(e) and "already in use" in str(e):
                 log.warning(
                     f"stale containers for app {name=}, removing and recreating"
                 )
-                await subprocess(
-                    "docker-compose", "down", cwd=get_installed_apps_path() / name
-                )
-                await subprocess(
-                    "docker-compose", "up", "-d", cwd=get_installed_apps_path() / name
-                )
+                await asyncio.to_thread(client.compose.down)
+                await asyncio.to_thread(client.compose.up, detach=True)
             else:
                 raise
         async with db_conn() as conn:
@@ -74,7 +66,8 @@ async def docker_stop_app(name: str, set_status: bool = True):
         app = await db_installed_apps.get_by_name(conn, name)
         app_status = app["status"] if app else None
     if app_status in [Status.RUNNING, Status.UNINSTALLING]:
-        await subprocess("docker-compose", "stop", cwd=get_installed_apps_path() / name)
+        client = DockerClient(compose_project_directory=get_installed_apps_path() / name)
+        await asyncio.to_thread(client.compose.stop)
         if set_status:
             async with db_conn() as conn:
                 await db_installed_apps.update_status(conn, name, Status.STOPPED)
@@ -88,7 +81,8 @@ async def docker_shutdown_app(name: str, set_status: bool = True, force: bool =
         app = await db_installed_apps.get_by_name(conn, name)
         app_status = app["status"] if app else None
     if force or app_status in [Status.STOPPED, Status.UNINSTALLING]:
-        await subprocess("docker-compose", "down", cwd=get_installed_apps_path() / name)
+        client = DockerClient(compose_project_directory=get_installed_apps_path() / name)
+        await asyncio.to_thread(client.compose.down)
         if set_status:
             async with db_conn() as conn:
                 await db_installed_apps.update_status(conn, name, Status.DOWN)
@@ -154,15 +148,15 @@ def enrich_installed_app_with_meta(installed_app: InstalledApp) -> InstalledAppW
 
 
 async def docker_prune_images(apply_filter=True):
-    command = ["docker", "image", "prune", "-fa"]
-    if apply_filter:
-        command.extend(["--filter", f"until={settings().apps.pruning.max_age}h"])
+    filters = {"until": f"{settings().apps.pruning.max_age}h"} if apply_filter else {}
     try:
-        stdout = await subprocess(*command)
-    except SubprocessError as e:
+        output = await asyncio.to_thread(
+            DockerClient().image.prune, all=True, filters=filters
+        )
+    except DockerException as e:
         log.error(f"failed to prune docker images: {e}")
         return
-    lines = stdout.splitlines()
+    lines = output.splitlines()
     log.info(f"docker images pruned, {lines[-1]}")
     return lines[-1]
 

shard_core/web/internal/app_error.py

@@ -3,9 +3,9 @@
 from functools import lru_cache
 from pathlib import Path
 
-import docker
 import jinja2
-from docker import errors as docker_errors
+from python_on_whales import DockerClient
+from python_on_whales.exceptions import NoSuchContainer
 from fastapi import APIRouter, Request
 from fastapi.responses import HTMLResponse
 from pydantic import BaseModel
@@ -79,8 +79,8 @@ async def get_splash_behaviour(request: Request):
 def get_container_status(app_name):
     docker_client = get_docker_client()
     try:
-        status = docker_client.containers.get(app_name).status
-    except docker_errors.NotFound:
+        status = docker_client.container.inspect(app_name).state.status
+    except NoSuchContainer:
         status = "unknown"
     return status
 
@@ -93,7 +93,7 @@ def get_app_name(request):
 
 @lru_cache()
 def get_docker_client():
-    return docker.from_env()
+    return DockerClient()
 
 
 @lru_cache()

shard_core/web/internal/call_peer.py

@@ -2,8 +2,7 @@
 from functools import lru_cache
 from typing import List
 
-import docker
-from docker.models.containers import Container
+from python_on_whales import DockerClient, Container
 from fastapi import APIRouter, Request
 from starlette.responses import StreamingResponse
 
@@ -33,14 +32,16 @@ async def call_peer(id_: str, rest: str, request: Request):
 def _get_app_for_ip_address(ip_address: str):
     # todo: test this
     docker_client = _get_docker_client()
-    docker_client.networks.get("portal")
-    containers: List[Container] = docker_client.containers.list()
+    docker_client.network.inspect("portal")
+    containers: List[Container] = docker_client.container.list()
     for c in containers:
-        if c.attrs["NetworkSettings"]["Networks"]["portal"]["IPAddress"] == ip_address:
+        networks = c.network_settings.networks or {}
+        portal_net = networks.get("portal")
+        if portal_net and portal_net.ip_address == ip_address:
             return c.name
     raise RuntimeError(f"no running container found for address {ip_address}")
 
 
 @lru_cache()
 def _get_docker_client():
-    return docker.from_env()
+    return DockerClient()

tests/test_app_installation.py

@@ -1,6 +1,6 @@
-import docker
 import pytest
-from docker.errors import NotFound
+from python_on_whales import DockerClient
+from python_on_whales.exceptions import NoSuchContainer
 from fastapi import status
 from httpx import AsyncClient
 
@@ -21,15 +21,15 @@ async def test_get_initial_apps(api_client: AsyncClient):
 
 
 async def test_install_app(api_client: AsyncClient):
-    docker_client = docker.from_env()
+    docker_client = DockerClient()
     app_name = "mock_app"
 
     response = await api_client.post(f"protected/apps/{app_name}")
     assert response.status_code == status.HTTP_201_CREATED
 
     await wait_until_app_installed(api_client, app_name)
 
-    docker_client.containers.get(app_name)
+    docker_client.container.inspect(app_name)
 
     response = (await api_client.get("protected/apps")).json()
     assert len(response) == 4
@@ -66,8 +66,8 @@ async def test_install_app_twice(api_client: AsyncClient):
 
 
 async def test_uninstall_app(api_client: AsyncClient):
-    docker_client = docker.from_env()
-    docker_client.containers.get("filebrowser")
+    docker_client = DockerClient()
+    docker_client.container.inspect("filebrowser")
 
     response = await api_client.delete("protected/apps/filebrowser")
     assert response.status_code == status.HTTP_204_NO_CONTENT
@@ -77,12 +77,12 @@ async def test_uninstall_app(api_client: AsyncClient):
     response = (await api_client.get("protected/apps")).json()
     assert len(response) == 2
 
-    with pytest.raises(NotFound):
-        docker_client.containers.get("filebrowser")
+    with pytest.raises(NoSuchContainer):
+        docker_client.container.inspect("filebrowser")
 
 
 async def test_uninstall_running_app(api_client: AsyncClient):
-    docker_client = docker.from_env()
+    docker_client = DockerClient()
     app_name = "mock_app"
 
     response = await api_client.post(f"protected/apps/{app_name}")
@@ -108,13 +108,13 @@ async def test_uninstall_running_app(api_client: AsyncClient):
     response = (await api_client.get("protected/apps")).json()
     assert len(response) == 3  # Initial apps are still installed
 
-    with pytest.raises(NotFound):
-        docker_client.containers.get(app_name)
+    with pytest.raises(NoSuchContainer):
+        docker_client.container.inspect(app_name)
 
 
 async def test_install_custom_app(api_client: AsyncClient):
     app_name = "mock_app"
-    docker_client = docker.from_env()
+    docker_client = DockerClient()
 
     app_zip = mock_app_store_path() / app_name / f"{app_name}.zip"
     with open(app_zip, "rb") as f:
@@ -127,7 +127,7 @@ async def test_install_custom_app(api_client: AsyncClient):
 
     await wait_until_app_installed(api_client, app_name)
 
-    docker_client.containers.get(app_name)
+    docker_client.container.inspect(app_name)
 
     response = (await api_client.get("protected/apps")).json()
     assert len(response) == 4

tests/test_app_lifecycle.py

@@ -1,20 +1,20 @@
-import docker
+from python_on_whales import DockerClient
 from fastapi import status
 
 from shard_core.data_model.app_meta import InstalledApp, Status
 from tests.util import retry_async, wait_until_app_installed
 
 
 async def test_app_starts_and_stops(requests_mock, api_client):
-    docker_client = docker.from_env()
+    docker_client = DockerClient()
     app_name = "quick_stop"
 
     response = await api_client.post(f"protected/apps/{app_name}")
     assert response.status_code == status.HTTP_201_CREATED
 
     await wait_until_app_installed(api_client, app_name)
 
-    assert docker_client.containers.get(app_name).status == "created"
+    assert docker_client.container.inspect(app_name).state.status == "created"
     assert (
         InstalledApp.model_validate(
             (await api_client.get(f"protected/apps/{app_name}")).json()
@@ -32,7 +32,7 @@ async def test_app_starts_and_stops(requests_mock, api_client):
     response.raise_for_status()
 
     async def assert_app_running():
-        assert docker_client.containers.get(app_name).status == "running"
+        assert docker_client.container.inspect(app_name).state.status == "running"
         assert (
             InstalledApp.model_validate(
                 (await api_client.get(f"protected/apps/{app_name}")).json()
@@ -41,7 +41,7 @@ async def assert_app_running():
         )
 
     async def assert_app_exited():
-        assert docker_client.containers.get(app_name).status == "exited"
+        assert docker_client.container.inspect(app_name).state.status == "exited"
         assert (
             InstalledApp.model_validate(
                 (await api_client.get(f"protected/apps/{app_name}")).json()
@@ -78,7 +78,7 @@ async def assert_app_exited():
 
 
 async def test_always_on_app_starts(requests_mock, api_client):
-    docker_client = docker.from_env()
+    docker_client = DockerClient()
     app_name = "always_on"
 
     response = await api_client.post(f"protected/apps/{app_name}")
@@ -87,7 +87,7 @@ async def test_always_on_app_starts(requests_mock, api_client):
     await wait_until_app_installed(api_client, app_name)
 
     async def assert_app_running():
-        assert docker_client.containers.get(app_name).status == "running"
+        assert docker_client.container.inspect(app_name).state.status == "running"
         assert (
             InstalledApp.model_validate(
                 (await api_client.get(f"protected/apps/{app_name}")).json()