Avoid using user-wide global podman socket on podcvd

.github/workflows/presubmit.yaml

@@ -519,6 +519,14 @@ jobs:
       run: |
         # Grant permission on devices as it's hard to grant on test env
         sudo chmod 666 /dev/kvm /dev/vhost-net /dev/vhost-vsock
+        # Define storage.conf, as podman cannot read image src location when home dir is changed.
+        GRAPHROOT=$(podman info --format '{{.Store.GraphRoot}}')
+        echo "[storage]" > $GITHUB_WORKSPACE/storage.conf
+        echo "graphroot = \"${GRAPHROOT}\"" >> $GITHUB_WORKSPACE/storage.conf
+        podman version
+        podman info
+        podman image list
+        cat $GITHUB_WORKSPACE/storage.conf
         cd e2etests
         bazel test \
           //cvd/cvd_powerwash_tests \
@@ -527,4 +535,7 @@ jobs:
           --test_env=HOME=$HOME \
           --test_env=USE_PODCVD=true \
           --test_env=XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR \
+          --test_env=CONTAINERS_STORAGE_CONF=$GITHUB_WORKSPACE/storage.conf \
+          --sandbox_writable_path="$GRAPHROOT" \
+          --sandbox_writable_path="$GITHUB_WORKSPACE" \
           --test_output=errors

container/src/libcfcontainer/cuttlefish_container.go

@@ -19,8 +19,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"os"
-	"path/filepath"
 	"strings"
 
 	"dario.cat/mergo"
@@ -200,8 +198,3 @@ func (m *CuttlefishContainerManagerImpl) StopAndRemoveContainer(ctx context.Cont
 	}
 	return errors.Join(errs...)
 }
-
-func RootlessPodmanSocketAddr() string {
-	socketPath := filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "podman/podman.sock")
-	return fmt.Sprintf("unix://%s", socketPath)
-}

container/src/podcvd/internal/const.go

@@ -19,6 +19,7 @@ var imageName = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orche
 const (
 	portOperatorHttps = 1443
 	ifName            = "podcvd"
+	podcvdRootDir     = "/var/tmp/podcvd"
 )
 
 const (

container/src/podcvd/internal/host.go

@@ -295,10 +295,6 @@ func createAndStartContainer(ccm libcfcontainer.CuttlefishContainerManager, comm
 	if productOut == "" {
 		productOut = currentDir
 	}
-	podcvdRootDir := "/var/tmp/podcvd"
-	if err := os.MkdirAll(podcvdRootDir, 0777); err != nil {
-		return "", fmt.Errorf("failed to create podcvd root dir: %w", err)
-	}
 	podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID)
 	if err := os.MkdirAll(podcvdHomeDir, 0755); err != nil {
 		return "", fmt.Errorf("failed to create podcvd home dir: %w", err)

container/src/podcvd/internal/main.go

@@ -20,12 +20,18 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"os"
+	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"strconv"
 	"sync"
+	"syscall"
+	"time"
 
 	"github.com/google/android-cuttlefish/container/src/libcfcontainer"
+	"github.com/google/uuid"
 )
 
 func Main(args []string) error {
@@ -34,7 +40,31 @@ func Main(args []string) error {
 		cvdArgs.SubCommandArgs = []string{"help"}
 	}
 
-	ccm, err := CuttlefishContainerManager()
+	podcvdSockDir := filepath.Join(podcvdRootDir, "sock")
+	if err := os.MkdirAll(podcvdSockDir, 0777); err != nil {
+		return fmt.Errorf("failed to create podcvd root dir: %w", err)
+	}
+	sockfilePath := filepath.Join(podcvdSockDir, fmt.Sprintf("podcvd_%s.sock", uuid.New().String()))
+	socketPath := fmt.Sprintf("unix://%s", sockfilePath)
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+	cmd := exec.Command("podman", "system", "service", "--time=0", socketPath)
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("failed to start podman system service: %w", err)
+	}
+	defer os.Remove(sockfilePath)
+	defer cmd.Process.Kill()
+	go func() {
+		<-sigChan
+		cmd.Process.Kill()
+		os.Remove(sockfilePath)
+		os.Exit(0)
+	}()
+	if err := waitSocketRunning(sockfilePath); err != nil {
+		return err
+	}
+	os.Setenv("DOCKER_HOST", socketPath)
+	ccm, err := CuttlefishContainerManager(socketPath)
 	if err != nil {
 		return err
 	}
@@ -90,6 +120,18 @@ func Main(args []string) error {
 	return nil
 }
 
+func waitSocketRunning(path string) error {
+	start := time.Now()
+	timeout := time.Second
+	for time.Since(start) < timeout {
+		if _, err := net.Dial("unix", path); err == nil {
+			return nil
+		}
+		time.Sleep(1 * time.Millisecond)
+	}
+	return fmt.Errorf("timed out waiting for podman socket to be ready")
+}
+
 func disconnectAdb(ccm libcfcontainer.CuttlefishContainerManager, groupName string) error {
 	var stdoutBuf bytes.Buffer
 	if err := ccm.ExecOnContainer(context.Background(), ContainerName(groupName), []string{"cvd", "fleet"}, nil, &stdoutBuf, nil); err != nil {
@@ -161,7 +203,7 @@ func handleSubcommandsForSingleInstanceGroup(ccm libcfcontainer.CuttlefishContai
 			return fmt.Errorf("failed to inspect container: %w", err)
 		}
 		attemptID := inspectRes.Config.Labels["attempt_id"]
-		podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID)
+		podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID)
 		UpdateCvdGroupJsonRaw(res, podcvdHomeDir, ip)
 		stdout, err := json.MarshalIndent(res, "", "        ")
 		if err != nil {
@@ -209,7 +251,7 @@ func clearAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro
 	for err := range errCh {
 		errs = append(errs, err)
 	}
-	uidDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()))
+	uidDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()))
 	if err := os.RemoveAll(uidDir); err != nil {
 		errs = append(errs, fmt.Errorf("failed to remove uid dir: %w", err))
 	}
@@ -249,7 +291,7 @@ func fleetAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro
 				return
 			}
 			attemptID := inspectRes.Config.Labels["attempt_id"]
-			podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID)
+			podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID)
 			for idx := range res.Groups {
 				UpdateCvdGroupJsonRaw(res.Groups[idx], podcvdHomeDir, ip)
 			}

container/src/podcvd/internal/util.go

@@ -26,9 +26,9 @@ import (
 	"github.com/docker/docker/api/types/filters"
 )
 
-func CuttlefishContainerManager() (libcfcontainer.CuttlefishContainerManager, error) {
+func CuttlefishContainerManager(sockAddr string) (libcfcontainer.CuttlefishContainerManager, error) {
 	ccmOpts := libcfcontainer.CuttlefishContainerManagerOpts{
-		SockAddr: libcfcontainer.RootlessPodmanSocketAddr(),
+		SockAddr: sockAddr,
 	}
 	return libcfcontainer.NewCuttlefishContainerManager(ccmOpts)
 }