diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index cdb5645cdc8..fb91a313ab3 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -519,6 +519,14 @@ jobs: run: | # Grant permission on devices as it's hard to grant on test env sudo chmod 666 /dev/kvm /dev/vhost-net /dev/vhost-vsock + # Define storage.conf, as podman cannot read image src location when home dir is changed. + GRAPHROOT=$(podman info --format '{{.Store.GraphRoot}}') + echo "[storage]" > $GITHUB_WORKSPACE/storage.conf + echo "graphroot = \"${GRAPHROOT}\"" >> $GITHUB_WORKSPACE/storage.conf + podman version + podman info + podman image list + cat $GITHUB_WORKSPACE/storage.conf cd e2etests bazel test \ //cvd/cvd_powerwash_tests \ @@ -527,4 +535,7 @@ jobs: --test_env=HOME=$HOME \ --test_env=USE_PODCVD=true \ --test_env=XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR \ + --test_env=CONTAINERS_STORAGE_CONF=$GITHUB_WORKSPACE/storage.conf \ + --sandbox_writable_path="$GRAPHROOT" \ + --sandbox_writable_path="$GITHUB_WORKSPACE" \ --test_output=errors diff --git a/container/src/libcfcontainer/cuttlefish_container.go b/container/src/libcfcontainer/cuttlefish_container.go index f60f6c907dd..c158919f604 100644 --- a/container/src/libcfcontainer/cuttlefish_container.go +++ b/container/src/libcfcontainer/cuttlefish_container.go @@ -19,8 +19,6 @@ import ( "errors" "fmt" "io" - "os" - "path/filepath" "strings" "dario.cat/mergo" @@ -200,8 +198,3 @@ func (m *CuttlefishContainerManagerImpl) StopAndRemoveContainer(ctx context.Cont } return errors.Join(errs...) } - -func RootlessPodmanSocketAddr() string { - socketPath := filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "podman/podman.sock") - return fmt.Sprintf("unix://%s", socketPath) -} diff --git a/container/src/podcvd/internal/const.go b/container/src/podcvd/internal/const.go index 6609c78e352..aa43aaf9cd5 100644 --- a/container/src/podcvd/internal/const.go +++ b/container/src/podcvd/internal/const.go @@ -19,6 +19,7 @@ var imageName = "us-docker.pkg.dev/android-cuttlefish-artifacts/cuttlefish-orche const ( portOperatorHttps = 1443 ifName = "podcvd" + podcvdRootDir = "/var/tmp/podcvd" ) const ( diff --git a/container/src/podcvd/internal/host.go b/container/src/podcvd/internal/host.go index e610d251272..d62c54148c0 100644 --- a/container/src/podcvd/internal/host.go +++ b/container/src/podcvd/internal/host.go @@ -295,10 +295,6 @@ func createAndStartContainer(ccm libcfcontainer.CuttlefishContainerManager, comm if productOut == "" { productOut = currentDir } - podcvdRootDir := "/var/tmp/podcvd" - if err := os.MkdirAll(podcvdRootDir, 0777); err != nil { - return "", fmt.Errorf("failed to create podcvd root dir: %w", err) - } podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) if err := os.MkdirAll(podcvdHomeDir, 0755); err != nil { return "", fmt.Errorf("failed to create podcvd home dir: %w", err) diff --git a/container/src/podcvd/internal/main.go b/container/src/podcvd/internal/main.go index 1dddc2f2be2..988d20adc1e 100644 --- a/container/src/podcvd/internal/main.go +++ b/container/src/podcvd/internal/main.go @@ -20,12 +20,18 @@ import ( "encoding/json" "errors" "fmt" + "net" "os" + "os/exec" + "os/signal" "path/filepath" "strconv" "sync" + "syscall" + "time" "github.com/google/android-cuttlefish/container/src/libcfcontainer" + "github.com/google/uuid" ) func Main(args []string) error { @@ -34,7 +40,31 @@ func Main(args []string) error { cvdArgs.SubCommandArgs = []string{"help"} } - ccm, err := CuttlefishContainerManager() + podcvdSockDir := filepath.Join(podcvdRootDir, "sock") + if err := os.MkdirAll(podcvdSockDir, 0777); err != nil { + return fmt.Errorf("failed to create podcvd root dir: %w", err) + } + sockfilePath := filepath.Join(podcvdSockDir, fmt.Sprintf("podcvd_%s.sock", uuid.New().String())) + socketPath := fmt.Sprintf("unix://%s", sockfilePath) + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + cmd := exec.Command("podman", "system", "service", "--time=0", socketPath) + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start podman system service: %w", err) + } + defer os.Remove(sockfilePath) + defer cmd.Process.Kill() + go func() { + <-sigChan + cmd.Process.Kill() + os.Remove(sockfilePath) + os.Exit(0) + }() + if err := waitSocketRunning(sockfilePath); err != nil { + return err + } + os.Setenv("DOCKER_HOST", socketPath) + ccm, err := CuttlefishContainerManager(socketPath) if err != nil { return err } @@ -90,6 +120,18 @@ func Main(args []string) error { return nil } +func waitSocketRunning(path string) error { + start := time.Now() + timeout := time.Second + for time.Since(start) < timeout { + if _, err := net.Dial("unix", path); err == nil { + return nil + } + time.Sleep(1 * time.Millisecond) + } + return fmt.Errorf("timed out waiting for podman socket to be ready") +} + func disconnectAdb(ccm libcfcontainer.CuttlefishContainerManager, groupName string) error { var stdoutBuf bytes.Buffer if err := ccm.ExecOnContainer(context.Background(), ContainerName(groupName), []string{"cvd", "fleet"}, nil, &stdoutBuf, nil); err != nil { @@ -161,7 +203,7 @@ func handleSubcommandsForSingleInstanceGroup(ccm libcfcontainer.CuttlefishContai return fmt.Errorf("failed to inspect container: %w", err) } attemptID := inspectRes.Config.Labels["attempt_id"] - podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID) + podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) UpdateCvdGroupJsonRaw(res, podcvdHomeDir, ip) stdout, err := json.MarshalIndent(res, "", " ") if err != nil { @@ -209,7 +251,7 @@ func clearAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro for err := range errCh { errs = append(errs, err) } - uidDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid())) + uidDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid())) if err := os.RemoveAll(uidDir); err != nil { errs = append(errs, fmt.Errorf("failed to remove uid dir: %w", err)) } @@ -249,7 +291,7 @@ func fleetAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro return } attemptID := inspectRes.Config.Labels["attempt_id"] - podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID) + podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID) for idx := range res.Groups { UpdateCvdGroupJsonRaw(res.Groups[idx], podcvdHomeDir, ip) } diff --git a/container/src/podcvd/internal/util.go b/container/src/podcvd/internal/util.go index f6ee801f5a3..62be5cce053 100644 --- a/container/src/podcvd/internal/util.go +++ b/container/src/podcvd/internal/util.go @@ -26,9 +26,9 @@ import ( "github.com/docker/docker/api/types/filters" ) -func CuttlefishContainerManager() (libcfcontainer.CuttlefishContainerManager, error) { +func CuttlefishContainerManager(sockAddr string) (libcfcontainer.CuttlefishContainerManager, error) { ccmOpts := libcfcontainer.CuttlefishContainerManagerOpts{ - SockAddr: libcfcontainer.RootlessPodmanSocketAddr(), + SockAddr: sockAddr, } return libcfcontainer.NewCuttlefishContainerManager(ccmOpts) }