Enhance MicroCeph orchestrator support

.github/workflows/tests.yml

@@ -183,11 +183,17 @@ jobs:
         with:
           go-version: 1.25.x
 
-      - name: Run unit-tests
+      - name: Run Go unit-tests
         run: |
           cd microceph
           make check-unit
 
+      - name: Run orchestrator Python unit-tests
+        run: |
+          python3 -m pip install ./microceph-orch pytest
+          cd microceph-orch
+          PYTHONPATH=src:tests python3 -m pytest tests/ -v
+
   single-system-tests:
     name: Single node with encryption
     runs-on: ubuntu-22.04
@@ -237,7 +243,7 @@ jobs:
           sudo microceph.ceph mgr module enable microceph
           sudo microceph.ceph orch set backend microceph
 
-      - name: Verify Orch module
+      - name: Verify Orch module (basic)
         run: |
           set -eux
           hn=$(hostname)
@@ -333,6 +339,11 @@ jobs:
       - name: Disable RGW
         run: ~/actionutils.sh disable_rgw
 
+      - name: Run orchestrator integration tests
+        run: |
+          set -eux
+          bash tests/scripts/test-orch.sh microceph.ceph
+
       - name: Enable RGW with SSL enabled
         run: ~/actionutils.sh enable_rgw_ssl
 
@@ -523,6 +534,16 @@ jobs:
       - name: Test RGW certificate rotation with --target (cross-node)
         run: ~/actionutils.sh headexec test_certificate_set_rgw_target node-wrk1
 
+      - name: Enable orchestrator and run integration tests
+        run: |
+          set -eux
+          lxc exec node-wrk0 -- sh -c "microceph.ceph mgr module enable microceph"
+          lxc exec node-wrk0 -- sh -c "microceph.ceph orch set backend microceph"
+          sleep 10
+          lxc exec node-wrk0 -- sh -c "microceph.ceph orch status"
+          lxc file push tests/scripts/test-orch.sh node-wrk0/tmp/test-orch.sh
+          lxc exec node-wrk0 -- bash /tmp/test-orch.sh microceph.ceph
+
       - name: Print logs for failure
         if: failure()
         run: |

.gitignore

@@ -5,3 +5,5 @@ _build
 .vscode
 microceph/coverage.out
 microceph/coverage.html
+__pycache__/
+*.pyc

microceph-orch/pyproject.toml

@@ -12,3 +12,8 @@ dependencies = [
 
 [tool.uv.sources]
 snap-helpers = { git = "https://github.com/albertodonato/snap-helpers" }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+]

microceph-orch/src/microceph/client/cluster.py

@@ -18,7 +18,7 @@ def get_cluster_members(self) -> list:
         """
         result = []
         cluster = self._get("/core/1.0/cluster")
-        members = cluster.get("metadata", {})
+        members = cluster.get("metadata") or []
         keys = ["name", "address", "status"]
         for member in members:
             result.append({k: v for k, v in member.items() if k in keys})
@@ -55,26 +55,85 @@ class ExtendedAPIService(service.BaseService):
     def list_services(self) -> list[dict]:
         """List all services."""
         services = self._get("/1.0/services")
-        return services.get("metadata")
-
-    def list_resources(self) -> list[dict]:
-        """List all resources."""
-        nodes = self._get("/1.0/resources")
-        return nodes.get("metadata")
+        return services.get("metadata") or []
 
     def list_disks(self) -> list[dict]:
         """List all disks"""
         disks = self._get("/1.0/disks")
-        return disks.get("metadata")
+        return disks.get("metadata") or []
 
     def get_status(self) -> dict[str, dict]:
         """Get status of the cluster."""
         cluster = self._get("/1.0/status")
-        members = cluster.get("metadata", {})
+        members = cluster.get("metadata") or []
         return {
             member["name"]: {
                 "status": member["status"],
                 "address": member["address"],
             }
             for member in members
         }
+
+    def enable_service(self, name: str, payload: str = "", wait: bool = True,
+                       target: str | None = None) -> None:
+        """Enable a service on the cluster.
+
+        Sends a PUT request to /1.0/services/<name> with an EnableService payload.
+        The Go API dispatches this to ServicePlacementHandler which runs the full
+        placement lifecycle: PopulateParams, HospitalityCheck, ServiceInit,
+        PostPlacementCheck, DbUpdate.
+
+        :param name: service name (e.g. 'rgw', 'nfs', 'rbd-mirror')
+        :param payload: JSON string with service-specific parameters
+        :param wait: if True, the server waits for the service to be fully enabled
+        :param target: optional cluster member name. When set, the
+            server-side proxyTarget middleware (microcluster) forwards
+            the request over mTLS to that node, allowing per-host
+            service enablement from the local unix socket. When None,
+            no target is forwarded and the server handles the request
+            on the node receiving the unix-socket connection.
+        """
+        # Note: The "bool" key maps to Go's EnableService.Wait field which has
+        # the struct tag `json:"bool"` (upstream naming quirk in MicroCeph).
+        params = {"target": target} if target is not None else None
+        self._put(f"/1.0/services/{name}", json={
+            "name": name,
+            "bool": wait,
+            "payload": payload,
+        }, params=params)
+
+    def delete_service(self, name: str, target: str | None = None) -> None:
+        """Delete/disable a service on the cluster.
+
+        :param name: service name (e.g. 'rgw', 'nfs', 'rbd-mirror')
+        :param target: optional cluster member name; see enable_service.
+        """
+        params = {"target": target} if target is not None else None
+        self._delete(f"/1.0/services/{name}", params=params)
+
+    def restart_services(self, services: list[str],
+                         target: str | None = None) -> None:
+        """Restart one or more services on the cluster.
+
+        Sends a POST to /1.0/services/restart with a list of service objects.
+
+        :param services: list of service names (e.g. ['mon', 'rgw'])
+        :param target: optional cluster member name; see enable_service.
+        """
+        params = {"target": target} if target is not None else None
+        payload = [{"service": svc} for svc in services]
+        self._post("/1.0/services/restart", json=payload, params=params)
+
+    def delete_nfs_service(self, cluster_id: str,
+                           target: str | None = None) -> None:
+        """Delete/disable an NFS service by cluster ID.
+
+        NFS deletion requires the cluster_id in the request body, unlike
+        other services which are identified by URL path alone.
+
+        :param cluster_id: NFS cluster identifier
+        :param target: optional cluster member name; see enable_service.
+        """
+        params = {"target": target} if target is not None else None
+        self._delete("/1.0/services/nfs",
+                     json={"cluster_id": cluster_id}, params=params)

microceph-orch/src/microceph/client/service.py

@@ -145,8 +145,13 @@ def _request(self, method, path, **kwargs):  # noqa: C901 too complex
         try:
             response.raise_for_status()
         except HTTPError as e:
-            # Do some nice translating to microclusterdexceptions
-            error = response.json().get("error")
+            # Do some nice translating to microclusterdexceptions.
+            # The error body is normally JSON with an "error" key, but a
+            # non-JSON body or a missing key must not mask the real HTTPError.
+            try:
+                error = response.json().get("error") or ""
+            except ValueError:
+                error = ""
             if "remote with name" in error:
                 raise NodeAlreadyExistsException(
                     "Already node exists in the microcluster"