feat: add Docker Compose deployment driver and image management

.github/workflows/lxc-test.yml

@@ -43,8 +43,9 @@ jobs:
         run: |
           i=1
           while IFS= read -r line || [ -n "$line" ]; do
-            [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
-            echo "cmd${i}=${line}" >> "$GITHUB_OUTPUT"
+            trimmed=$(echo "$line" | xargs)
+            [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue
+            echo "cmd${i}=${trimmed}" >> "$GITHUB_OUTPUT"
             i=$((i+1))
           done <<'EOF'
           ${{ inputs.cmlxc_commands }}
@@ -68,23 +69,25 @@ jobs:
           repository: chatmail/cmlxc
           ref: ${{ inputs.cmlxc_version }}
           path: cmlxc
-          fetch-depth: 0
 
       - name: Install Incus (Zabbly)
         run: |
           sudo mkdir -p /etc/apt/keyrings
           sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc
           echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list
           sudo apt-get update
-          sudo apt-get install -y incus=1:7.2* incus-client=1:7.2* incus-base=1:7.2*
-          incus --version
+          sudo apt-get install -y incus-client=1:7.2* incus-base=1:7.2*
 
       - name: Initialise Incus
         run: |
           sudo systemctl stop docker.socket docker || true
           sudo iptables -P FORWARD ACCEPT
           sudo sysctl -w fs.inotify.max_user_instances=65535
           sudo sysctl -w fs.inotify.max_user_watches=65535
+          # Disable AppArmor restrictions so Docker-in-LXC containers
+          # can run systemd (needs cgroup notification socket access).
+          sudo systemctl stop apparmor || true
+          sudo apparmor_parser -R /etc/apparmor.d/* 2>/dev/null || true
           sudo incus admin init --auto
           sudo chmod 666 /var/lib/incus/unix.socket
           sudo chown -R $USER:$USER ~/.config/incus
@@ -99,17 +102,17 @@ jobs:
           python -m pip install --upgrade pip
           pip install ./cmlxc
 
-      - name: Cache Incus images
+      - name: Restore Incus image cache
         id: cache-images
-        uses: actions/cache@v5
+        uses: actions/cache/restore@v5
         with:
           path: /tmp/incus-cache
           key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }}
 
       - name: Import cached images
         run: |
           mkdir -p /tmp/incus-cache
-          for alias in localchat-base localchat-builder; do
+          for alias in localchat-base localchat-builder localchat-cmdeploy localchat-docker; do
             if [ -f /tmp/incus-cache/$alias.tar.gz ]; then
               echo "Importing: $alias"
               incus image import /tmp/incus-cache/$alias.tar.gz --alias $alias || true
@@ -172,12 +175,12 @@ jobs:
           set -eu
           i=0
           while IFS= read -r cmd || [ -n "$cmd" ]; do
-            [[ "$cmd" =~ ^[[:space:]]*(#|$) ]] && continue
+            trimmed=$(echo "$cmd" | xargs)
+            [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue
             i=$((i+1))
             if [ $i -le 12 ]; then continue; fi
-
-            echo "::group::Run: $cmd"
-            eval "$cmd" || { echo "::endgroup::"; exit 1; }
+            echo "::group::Run: $trimmed"
+            eval "$trimmed" || { echo "::endgroup::"; exit 1; }
             echo "::endgroup::"
           done <<< "$CMLXC_COMMANDS"
 
@@ -186,26 +189,73 @@ jobs:
         run: |
           for c in $(incus list -c n --format csv); do
             echo "::group::Logs for $c"
-            incus exec "$c" -- journalctl -p warning --no-pager -n 100 || true
+            incus exec "$c" -- journalctl --no-pager -n 200 || true
+            # Dump Docker container logs if present
+            svc=chatmail
+            if incus exec "$c" -- docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "$svc"; then
+              echo "--- docker logs $svc ---"
+              incus exec "$c" -- docker logs "$svc" --tail 200 2>&1 || true
+              echo "--- dovecot journal ---"
+              incus exec "$c" -- docker exec "$svc" journalctl -u dovecot --no-pager -n 50 2>&1 || true
+              echo "--- postfix journal ---"
+              incus exec "$c" -- docker exec "$svc" journalctl -u postfix --no-pager -n 50 2>&1 || true
+              echo "--- failed units ---"
+              incus exec "$c" -- docker exec "$svc" systemctl --failed --no-pager 2>&1 || true
+              echo "--- dovecot -n (effective config) ---"
+              incus exec "$c" -- docker exec "$svc" dovecot -n 2>&1 | tail -40 || true
+              echo "--- ssl cert check ---"
+              incus exec "$c" -- docker exec "$svc" ls -la /etc/ssl/certs/mailserver.pem /etc/ssl/private/mailserver.key 2>&1 || true
+            fi
             echo "::endgroup::"
           done
 
       - name: Export images for cache
         if: always() && steps.cache-images.outputs.cache-hit != 'true'
         run: |
           mkdir -p /tmp/incus-cache
+          # Publish the builder LXC container as a cached image (the Docker
+          # container inside gets recreated on compose up, so the LXC is clean).
+          # Only skip localchat-cmdeploy on failure -- it bakes deploy state
+          # directly into the LXC and would carry broken config into the next run.
           if incus list -c n --format csv | grep -q builder-localchat; then
-            incus exec builder-localchat -- rm -rf /root/relays /root/minitest-venv /root/.ssh/config*
+            echo "Cleaning up builder container before publishing ..."
+            incus exec builder-localchat -- bash -c 'rm -rf /root/relays/* /root/.cache/* /root/.npm /root/.bun'
             echo "Publishing builder container as image ..."
             incus publish builder-localchat --alias localchat-builder --force || true
           fi
-          for alias in localchat-base localchat-builder; do
+          # Publish Docker relay container with engine only (strip images to keep cache small)
+          for ct in $(incus list -c n --format csv | grep -v builder); do
+            if incus exec "$ct" -- docker info >/dev/null 2>&1; then
+              echo "Stripping Docker images from $ct ..."
+              incus exec "$ct" -- docker system prune -af --volumes 2>/dev/null || true
+              echo "Publishing $ct as localchat-docker ..."
+              incus publish "$ct" --alias localchat-docker --force || true
+              break
+            fi
+          done
+          exported=0
+          if [ "${{ job.status }}" = "success" ]; then
+            aliases="localchat-base localchat-builder localchat-cmdeploy localchat-docker"
+          else
+            aliases="localchat-base localchat-builder localchat-docker"
+          fi
+          for alias in $aliases; do
             if incus image list --format csv -c l | grep -q "^$alias$"; then
               echo "Exporting: $alias"
               incus image export $alias /tmp/incus-cache/$alias || true
               if [ -f /tmp/incus-cache/$alias ] && [ ! -f /tmp/incus-cache/$alias.tar.gz ]; then
                 mv /tmp/incus-cache/$alias /tmp/incus-cache/$alias.tar.gz
               fi
+              exported=$((exported+1))
             fi
           done
+          echo "exported=$exported" >> "$GITHUB_OUTPUT"
+        id: export-images
+
+      - name: Save Incus image cache
+        if: always() && steps.export-images.outputs.exported > 0
+        uses: actions/cache/save@v5
+        with:
+          path: /tmp/incus-cache
+          key: incus-v5-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }}
 

.github/workflows/test.yml

@@ -28,8 +28,7 @@ jobs:
         sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc
         echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list
         sudo apt-get update
-        sudo apt-get install -y incus=1:7.2* incus-client=1:7.2* incus-base=1:7.2*
-        incus --version
+        sudo apt-get install -y incus-client=1:7.2* incus-base=1:7.2*
 
     - name: Initialise Incus
       run: |

README.md

@@ -81,6 +81,34 @@ Each `deploy-*` invocation initialises the driver's source in the
 builder (wipe-and-reclone).
 
 
+**Deploy via Docker Compose** (runs chatmail inside Docker-in-LXC):
+
+    # Pull a pre-built image directly from GHCR
+    cmlxc docker deploy --source ghcr:main dk0
+    cmlxc docker deploy --source ghcr:sha-ce05b26 dk0
+
+    # Load a local image tarball
+    cmlxc docker deploy --image ./chatmail.tar dk0
+
+    # Inject a locally-built image from the host Docker daemon
+    cmlxc docker deploy --source docker:chatmail-relay:latest dk0
+
+Pull a newer image into an already-deployed relay:
+
+    cmlxc docker pull dk0
+    cmlxc docker pull dk0 --tag sha-ce05b26
+
+Inspect running services and logs:
+
+    cmlxc docker ps dk0
+    cmlxc docker logs dk0
+    cmlxc docker logs dk0 -f
+
+SSH into a Docker service (auto-configured by ``cmlxc``):
+
+    ssh chatmail@dk0.localchat
+
+
 **Run integration tests** inside the builder:
 
     cmlxc test-mini cm0
@@ -167,13 +195,14 @@ the host only needs `cmlxc` itself.
 
 **Relay containers** (e.g. `cm0-localchat`, `mad1-localchat`) --
 ephemeral containers that receive a deployed chatmail service.
-Each relay is locked to a single deployment driver (`cmdeploy` or
-`madmail`); switching requires destroying and re-creating the container.
+Each relay is locked to a single deployment driver (`cmdeploy`,
+`madmail`, or `docker`); switching requires destroying and re-creating
+the container.
 
 
 ### Deployment drivers
 
-Drivers live in `driver_cmdeploy.py` and `driver_madmail.py`.
+Drivers live in `driver_cmdeploy.py`, `driver_madmail.py`, and `driver_docker.py`.
 Each driver module exports its CLI subcommand metadata,
 builder init, and deploy orchestration.
 `cli.py` generates the `deploy-*` subcommands from a `DRIVER_BY_NAME` mapping.
@@ -187,6 +216,45 @@ builder init, and deploy orchestration.
   pushes it via SCP and runs `madmail install --simple --ip <IP>`.
   No DNS entries are needed.
 
+- **docker** -- deploys chatmail via Docker Compose inside a Docker-in-LXC
+relay container (`security.nesting=true`), either directly pulled from GHCR or
+injected from a host docker instance. Docker is installed inside the relay
+automatically; no host Docker installation is required.
+
+#### Docker subcommands
+
+- `docker deploy RELAY` -- deploy chatmail into a relay container via
+  Docker Compose.  Three image sources are supported:
+  - `--source ghcr:TAG` -- pull a pre-built image from GHCR directly
+    into the relay.  No builder container is involved.
+  - `--source docker:TAG` -- pipe a locally-built image from the host
+    Docker daemon into the relay via `docker save | docker load`.
+  - `--image PATH` -- load a pre-exported image tarball.
+  A docker-compose.yaml is fetched from
+  [chatmail/docker](https://github.com/chatmail/docker) unless
+  `--compose URL` overrides the source.
+
+- `docker pull RELAY` -- pull a newer image from GHCR into an already
+  deployed relay without a full redeploy.  Use `--tag` to specify the
+  image tag (default: `main`).
+
+- `docker ps RELAY` -- list running Docker Compose services in a relay.
+
+- `docker logs RELAY` -- show Docker Compose logs (last 100 lines).
+  Pass `-f` to follow in real time.
+
+- `docker shell RELAY [SERVICE]` -- open an interactive shell inside
+  the named Compose service (default: `chatmail`).
+
+#### SSH into Docker services
+
+For Docker-deployed relays, `cmlxc` auto-generates SSH config entries for
+each running Compose service.  After any deploy or `cmlxc status`, you can:
+
+    ssh chatmail@dk0.localchat
+
+This uses `ProxyCommand` to run `docker exec` inside the LXC container.
+
 
 ## Releasing
 

src/cmlxc/cli.py

@@ -1,10 +1,11 @@
 """cmlxc -- Manage local chatmail relay containers via Incus.
 
 Standard workflow:
-init -> deploy-cmdeploy/deploy-madmail -> test-cmdeploy/test-madmail/test-mini.
+init -> deploy-cmdeploy/deploy-madmail/docker deploy -> test-*/test-mini.
 """
 
 import argparse
+import os
 import subprocess
 from pathlib import Path
 
@@ -21,6 +22,7 @@
 )
 from cmlxc.driver_base import __version__
 from cmlxc.driver_cmdeploy import CmdeployDriver
+from cmlxc.driver_docker import DockerDriver
 from cmlxc.driver_madmail import MadmailDriver, print_admin_info
 from cmlxc.incus import Incus, _is_ip_address, check_cgroup_compat
 from cmlxc.output import Out
@@ -41,15 +43,7 @@ def _container_completer(prefix, **kwargs):
 
 
 def _check_init(ix, out):
-    managed = ix.list_managed()
-    dns_running = any(
-        c["name"] == DNS_CONTAINER_NAME and c["status"] == "Running" for c in managed
-    )
-    if not dns_running or not ix.find_image([BASE_IMAGE_ALIAS]):
-        out.red("Error: cmlxc environment not initialized.")
-        out.red("Please run 'cmlxc init' first to set up the base image and DNS.")
-        return False
-    return True
+    return ix.check_init()
 
 
 def _destroy_all(ix, out):
@@ -266,13 +260,25 @@ def _add_test_relay_args(parser):
 
 def test_cmdeploy_cmd_options(parser):
     _add_test_relay_args(parser)
+    parser.add_argument(
+        "--relay-ref",
+        default=None,
+        help="Override relay git ref for tests (default: SHA from deployed image label).",
+    )
 
 
 def test_cmdeploy_cmd(args, out):
     """Run cmdeploy integration tests inside the builder container."""
     ix = Incus(out)
     ct = ix.get_running_relay(args.relay)
-    driver = CmdeployDriver(ct, out)
+    drv_cls = DRIVER_BY_NAME.get(ct.driver_name)
+    if drv_cls is None:
+        out.red(
+            f"Warning: unknown driver {ct.driver_name!r} for"
+            f" {ct.shortname}, falling back to cmdeploy."
+        )
+        drv_cls = CmdeployDriver
+    driver = drv_cls(ct, out)
     if not driver.check_init():
         return 1
 
@@ -302,6 +308,8 @@ def test_cmdeploy_cmd(args, out):
         drv2 = DRIVER_BY_NAME[ct2.driver_name](ct2, out)
         second_domain = drv2.get_test_domain_or_ip()
 
+    if args.relay_ref is not None:
+        os.environ["RELAY_REF"] = args.relay_ref
     return driver.run_tests(second_domain=second_domain)
 
 
@@ -527,11 +535,16 @@ def _print_container_status(out, c, ix):
 
 def _print_builder_repos(out, ct):
     try:
-        for name in DRIVER_BY_NAME:
-            path = f"/root/{name}-git-main"
+        seen = set()
+        for name, drv_cls in DRIVER_BY_NAME.items():
+            repo = drv_cls.REPO_NAME
+            if repo in seen:
+                continue
+            seen.add(repo)
+            path = f"/root/{repo}-git-main"
             status = ct.get_repo_status(path)
             if status:
-                out.print(f"{name}: {status}")
+                out.print(f"{repo}: {status}")
     except Exception:
         out.print("repos: (unavailable)")
 
@@ -612,7 +625,11 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False):
     ("destroy", destroy_cmd, destroy_cmd_options),
 ]
 
-DRIVER_BY_NAME = {"cmdeploy": CmdeployDriver, "madmail": MadmailDriver}
+DRIVER_BY_NAME = {
+    "cmdeploy": CmdeployDriver,
+    "docker": DockerDriver,
+    "madmail": MadmailDriver,
+}
 
 
 def _add_subcommand(subparsers, name, func, addopts, shared):
@@ -677,6 +694,10 @@ def main(args=None):
     if args.func is None:
         return parser.parse_args(["-h"])
 
+    # GitHub Actions: auto-enable max verbosity when debug logging is on
+    if not args.verbose and os.environ.get("RUNNER_DEBUG") == "1":
+        args.verbose = 3
+
     out = Out(verbosity=args.verbose)
     try:
         res = args.func(args, out)