From 462570affde396deb4af9af720f06b58466e644b Mon Sep 17 00:00:00 2001 From: Seungjae Yoo Date: Thu, 7 May 2026 10:51:27 +0900 Subject: [PATCH 1/2] Avoid using user-wide global podman socket on podcvd --- .github/workflows/presubmit.yaml | 4 ++ .../libcfcontainer/cuttlefish_container.go | 7 --- container/src/podcvd/internal/const.go | 1 + container/src/podcvd/internal/host.go | 4 -- container/src/podcvd/internal/main.go | 49 +++++++++++++++++-- container/src/podcvd/internal/util.go | 4 +- 6 files changed, 52 insertions(+), 17 deletions(-) diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index cdb5645cdc8..90f6acbeae7 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -519,6 +519,9 @@ jobs: run: | # Grant permission on devices as it's hard to grant on test env sudo chmod 666 /dev/kvm /dev/vhost-net /dev/vhost-vsock + # Define storage.conf, as podman cannot read image src location when home dir is changed. + echo "[storage]" > $HOME/storage.conf + echo "graphroot = \"$(podman info --format '{{.Store.GraphRoot}}')\"" >> $HOME/storage.conf cd e2etests bazel test \ //cvd/cvd_powerwash_tests \ @@ -527,4 +530,5 @@ jobs: --test_env=HOME=$HOME \ --test_env=USE_PODCVD=true \ --test_env=XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR \ + --test_env=CONTAINERS_STORAGE_CONF=$HOME/storage.conf \ --test_output=errors diff --git a/container/src/libcfcontainer/cuttlefish_container.go b/container/src/libcfcontainer/cuttlefish_container.go index f60f6c907dd..c158919f604 100644 --- a/container/src/libcfcontainer/cuttlefish_container.go +++ b/container/src/libcfcontainer/cuttlefish_container.go @@ -19,8 +19,6 @@ import ( "errors" "fmt" "io" - "os" - "path/filepath" "strings" "dario.cat/mergo" @@ -200,8 +198,3 @@ func (m *CuttlefishContainerManagerImpl) StopAndRemoveContainer(ctx context.Cont } return errors.Join(errs...) } - -func RootlessPodmanSocketAddr() string { - socketPath := filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "podman/podman.sock") - return fmt.Sprintf("unix://%s", socketPath) -} diff --git a/container/src/podcvd/internal/const.go b/container/src/podcvd/internal/const.go index 6609c78e352..aa43aaf9cd5 100644 --- a/container/src/podcvd/internal/const.go +++ b/container/src/podcvd/internal/const.go @@ -19,6 +19,7 @@ var imageName = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orche const ( portOperatorHttps = 1443 ifName = "podcvd" + podcvdRootDir = "/var/tmp/podcvd" ) const ( diff --git a/container/src/podcvd/internal/host.go b/container/src/podcvd/internal/host.go index e610d251272..d62c54148c0 100644 --- a/container/src/podcvd/internal/host.go +++ b/container/src/podcvd/internal/host.go @@ -295,10 +295,6 @@ func createAndStartContainer(ccm libcfcontainer.CuttlefishContainerManager, comm if productOut == "" { productOut = currentDir } - podcvdRootDir := "/var/tmp/podcvd" - if err := os.MkdirAll(podcvdRootDir, 0777); err != nil { - return "", fmt.Errorf("failed to create podcvd root dir: %w", err) - } podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) if err := os.MkdirAll(podcvdHomeDir, 0755); err != nil { return "", fmt.Errorf("failed to create podcvd home dir: %w", err) diff --git a/container/src/podcvd/internal/main.go b/container/src/podcvd/internal/main.go index 1dddc2f2be2..12ca45c2161 100644 --- a/container/src/podcvd/internal/main.go +++ b/container/src/podcvd/internal/main.go @@ -20,12 +20,18 @@ import ( "encoding/json" "errors" "fmt" + "net" "os" + "os/exec" + "os/signal" "path/filepath" "strconv" "sync" + "syscall" + "time" "github.com/google/android-cuttlefish/container/src/libcfcontainer" + "github.com/google/uuid" ) func Main(args []string) error { @@ -34,7 +40,30 @@ func Main(args []string) error { cvdArgs.SubCommandArgs = []string{"help"} } - ccm, err := CuttlefishContainerManager() + podcvdSockDir := filepath.Join(podcvdRootDir, "sock") + if err := os.MkdirAll(podcvdSockDir, 0777); err != nil { + return fmt.Errorf("failed to create podcvd root dir: %w", err) + } + sockfilePath := filepath.Join(podcvdSockDir, fmt.Sprintf("podcvd_%s.sock", uuid.New().String())) + socketPath := fmt.Sprintf("unix://%s", sockfilePath) + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + cmd := exec.Command("podman", "system", "service", "--time=0", socketPath) + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start podman system service: %w", err) + } + defer os.Remove(sockfilePath) + defer cmd.Process.Kill() + go func() { + <-sigChan + cmd.Process.Kill() + os.Remove(sockfilePath) + os.Exit(0) + }() + if err := waitSocketRunning(sockfilePath); err != nil { + return err + } + ccm, err := CuttlefishContainerManager(socketPath) if err != nil { return err } @@ -90,6 +119,18 @@ func Main(args []string) error { return nil } +func waitSocketRunning(path string) error { + start := time.Now() + timeout := time.Second + for time.Since(start) < timeout { + if _, err := net.Dial("unix", path); err == nil { + return nil + } + time.Sleep(1 * time.Millisecond) + } + return fmt.Errorf("timed out waiting for podman socket to be ready") +} + func disconnectAdb(ccm libcfcontainer.CuttlefishContainerManager, groupName string) error { var stdoutBuf bytes.Buffer if err := ccm.ExecOnContainer(context.Background(), ContainerName(groupName), []string{"cvd", "fleet"}, nil, &stdoutBuf, nil); err != nil { @@ -161,7 +202,7 @@ func handleSubcommandsForSingleInstanceGroup(ccm libcfcontainer.CuttlefishContai return fmt.Errorf("failed to inspect container: %w", err) } attemptID := inspectRes.Config.Labels["attempt_id"] - podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID) + podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) UpdateCvdGroupJsonRaw(res, podcvdHomeDir, ip) stdout, err := json.MarshalIndent(res, "", " ") if err != nil { @@ -209,7 +250,7 @@ func clearAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro for err := range errCh { errs = append(errs, err) } - uidDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid())) + uidDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid())) if err := os.RemoveAll(uidDir); err != nil { errs = append(errs, fmt.Errorf("failed to remove uid dir: %w", err)) } @@ -249,7 +290,7 @@ func fleetAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro return } attemptID := inspectRes.Config.Labels["attempt_id"] - podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID) + podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) for idx := range res.Groups { UpdateCvdGroupJsonRaw(res.Groups[idx], podcvdHomeDir, ip) } diff --git a/container/src/podcvd/internal/util.go b/container/src/podcvd/internal/util.go index f6ee801f5a3..62be5cce053 100644 --- a/container/src/podcvd/internal/util.go +++ b/container/src/podcvd/internal/util.go @@ -26,9 +26,9 @@ import ( "github.com/docker/docker/api/types/filters" ) -func CuttlefishContainerManager() (libcfcontainer.CuttlefishContainerManager, error) { +func CuttlefishContainerManager(sockAddr string) (libcfcontainer.CuttlefishContainerManager, error) { ccmOpts := libcfcontainer.CuttlefishContainerManagerOpts{ - SockAddr: libcfcontainer.RootlessPodmanSocketAddr(), + SockAddr: sockAddr, } return libcfcontainer.NewCuttlefishContainerManager(ccmOpts) } From aa3d56f2e5a909317b6c46b5f65c53e29df3dd41 Mon Sep 17 00:00:00 2001 From: Seungjae Yoo Date: Thu, 7 May 2026 10:51:27 +0900 Subject: [PATCH 2/2] WIP: Add more information to debug on E2E test --- .github/workflows/presubmit.yaml | 13 ++++++++++--- container/src/podcvd/internal/main.go | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 90f6acbeae7..fb91a313ab3 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -520,8 +520,13 @@ jobs: # Grant permission on devices as it's hard to grant on test env sudo chmod 666 /dev/kvm /dev/vhost-net /dev/vhost-vsock # Define storage.conf, as podman cannot read image src location when home dir is changed. - echo "[storage]" > $HOME/storage.conf - echo "graphroot = \"$(podman info --format '{{.Store.GraphRoot}}')\"" >> $HOME/storage.conf + GRAPHROOT=$(podman info --format '{{.Store.GraphRoot}}') + echo "[storage]" > $GITHUB_WORKSPACE/storage.conf + echo "graphroot = \"${GRAPHROOT}\"" >> $GITHUB_WORKSPACE/storage.conf + podman version + podman info + podman image list + cat $GITHUB_WORKSPACE/storage.conf cd e2etests bazel test \ //cvd/cvd_powerwash_tests \ @@ -530,5 +535,7 @@ jobs: --test_env=HOME=$HOME \ --test_env=USE_PODCVD=true \ --test_env=XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR \ - --test_env=CONTAINERS_STORAGE_CONF=$HOME/storage.conf \ + --test_env=CONTAINERS_STORAGE_CONF=$GITHUB_WORKSPACE/storage.conf \ + --sandbox_writable_path="$GRAPHROOT" \ + --sandbox_writable_path="$GITHUB_WORKSPACE" \ --test_output=errors diff --git a/container/src/podcvd/internal/main.go b/container/src/podcvd/internal/main.go index 12ca45c2161..988d20adc1e 100644 --- a/container/src/podcvd/internal/main.go +++ b/container/src/podcvd/internal/main.go @@ -63,6 +63,7 @@ func Main(args []string) error { if err := waitSocketRunning(sockfilePath); err != nil { return err } + os.Setenv("DOCKER_HOST", socketPath) ccm, err := CuttlefishContainerManager(socketPath) if err != nil { return err