diff --git a/libcontainer/cgroups/utils.go b/libcontainer/cgroups/utils.go
index a05945cba6c..421bf39e1a6 100644
--- a/libcontainer/cgroups/utils.go
+++ b/libcontainer/cgroups/utils.go
@@ -230,6 +230,11 @@ func rmdir(path string, retry bool) error {
 	tries := 10
 
 again:
+	// If we remove a non-exist dir in a ro mount point, it will
+	// return EROFS in `unix.Rmdir`, so we need to check first.
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil
+	}
 	err := unix.Rmdir(path)
 	switch err { // nolint:errorlint // unix errors are bare
 	case nil, unix.ENOENT:
@@ -257,15 +262,22 @@ func RemovePath(path string) error {
 	}
 
 	infos, err := os.ReadDir(path)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Please keep this error eraser, or else it will return ErrNotExist
+			// for cgroupv2.
+			// Please see https://github.com/opencontainers/runc/issues/4518
+			return nil
+		}
 		return err
 	}
 	for _, info := range infos {
-		if info.IsDir() {
-			// We should remove subcgroup first.
-			if err = RemovePath(filepath.Join(path, info.Name())); err != nil {
-				break
-			}
+		if !info.IsDir() {
+			continue
+		}
+		// We should remove subcgroup first.
+		if err = RemovePath(filepath.Join(path, info.Name())); err != nil {
+			return err
 		}
 	}
 	if err == nil {