From 40d3c3b9b028d408e975a0292bd4f614ff4bd65a Mon Sep 17 00:00:00 2001 From: Wolfgang Pross Date: Tue, 26 Sep 2023 10:19:33 +0000 Subject: [PATCH 1/6] Add Intel RDT support Add --rdt-class=COS to the create and run command to enable the assignment of a container to a Class of Service (COS). The COS represents a part of the cache based on the Cache Allocation Technology (CAT) feature that is part of Intel's Resource Director Technology (Intel RDT) feature set. By assigning a container to a COS, all PID's of the container have only access to the cache space defined for this COS. The COS has to be pre-configured based on the resctrl kernel driver. cat_l2 and cat_l3 flags in /proc/cpuinfo represent CAT support for cache level 2 and 3 respectively. Signed-off-by: Wolfgang Pross --- cmd/podman/common/create.go | 8 ++++++++ libpod/container_inspect.go | 7 +++++++ libpod/define/container_inspect.go | 1 + pkg/domain/entities/pods.go | 1 + pkg/specgen/generate/oci_linux.go | 6 ++++++ pkg/specgen/specgen.go | 4 ++++ pkg/specgenutil/specgen.go | 6 ++++++ 7 files changed, 33 insertions(+) diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index 5c4d903c7ed..3af1f1f15c2 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -370,6 +370,14 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, "quiet", "q", false, "Suppress output information when pulling images", ) + rdtClassFlagName := "rdt-class" + createFlags.StringVar( + &cf.IntelRdtClosID, + rdtClassFlagName, cf.IntelRdtClosID, + "Class of Service (COS) that the container should be assigned to", + ) + _ = cmd.RegisterFlagCompletionFunc(rdtClassFlagName, AutocompletePullOption) + createFlags.BoolVar( &cf.ReadOnly, "read-only", podmanConfig.ContainersConfDefaultsRO.Containers.ReadOnly, diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index bb6fc7f543a..736c1914667 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -198,6 +198,13 @@ func (c *Container) getContainerInspectData(size bool, driverData *define.Driver } } + if ctrSpec.Linux.IntelRdt != nil { + if ctrSpec.Linux.IntelRdt.ClosID != "" { + // container is assigned to a ClosID + data.State.IntelRdtClosID = ctrSpec.Linux.IntelRdt.ClosID + } + } + networkConfig, err := c.getContainerNetworkInfo() if err != nil { return nil, err diff --git a/libpod/define/container_inspect.go b/libpod/define/container_inspect.go index de4f700fa47..78b3a2aedaf 100644 --- a/libpod/define/container_inspect.go +++ b/libpod/define/container_inspect.go @@ -229,6 +229,7 @@ type InspectContainerState struct { RestoreLog string `json:"RestoreLog,omitempty"` Restored bool `json:"Restored,omitempty"` StoppedByUser bool `json:"StoppedByUser,omitempty"` + IntelRdtClosID string `json:"IntelRdtClosID,omitempty"` } // Healthcheck returns the HealthCheckResults. This is used for old podman compat diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 5c882a5ed3a..c6bdc0f5531 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -224,6 +224,7 @@ type ContainerCreateOptions struct { Init bool InitContainerType string InitPath string + IntelRdtClosID string Interactive bool IPC string Label []string diff --git a/pkg/specgen/generate/oci_linux.go b/pkg/specgen/generate/oci_linux.go index 6dc1dc28bbf..fa8cd4c24dd 100644 --- a/pkg/specgen/generate/oci_linux.go +++ b/pkg/specgen/generate/oci_linux.go @@ -216,6 +216,12 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt g.AddAnnotation(key, val) } + if s.IntelRdt != nil { + if s.IntelRdt.ClosID != "" { + g.SetLinuxIntelRdtClosID(s.IntelRdt.ClosID) + } + } + if s.ResourceLimits != nil { out, err := json.Marshal(s.ResourceLimits) if err != nil { diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index eca9a9bcc3d..9568ddc0484 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -514,6 +514,10 @@ type ContainerNetworkConfig struct { // ContainerResourceConfig contains information on container resource limits. type ContainerResourceConfig struct { + // IntelRdt defines the Intel RDT CAT Class of Service (COS) that all processes + // of the container should run in. + // Optional. + IntelRdt *spec.LinuxIntelRdt `json:"intelRdt,omitempty"` // ResourceLimits are resource limits to apply to the container., // Can only be set as root on cgroups v1 systems, but can be set as // rootless as well for cgroups v2. diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index 3f6475a7e4d..ea4fc4ca916 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -491,6 +491,12 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions s.Labels = labels } + // Intel RDT CAT + s.IntelRdt = &specs.LinuxIntelRdt{} + if c.IntelRdtClosID != "" { + s.IntelRdt.ClosID = c.IntelRdtClosID + } + // ANNOTATIONS annotations := make(map[string]string) From 24b2f829b9e1115b39899a33cf9ffc303a515aa1 Mon Sep 17 00:00:00 2001 From: Wolfgang Pross Date: Tue, 26 Sep 2023 10:45:17 +0000 Subject: [PATCH 2/6] Add test for Intel RDT support Signed-off-by: Wolfgang Pross --- test/e2e/create_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index cfd9d6e7ef0..10d0de2373a 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -75,6 +75,18 @@ var _ = Describe("Podman create", func() { Expect(session).Should(Exit(125)) }) + It("podman create adds rdt-class", func() { + session := podmanTest.Podman([]string{"create", "--rdt-class", "COS1", "--name", "rdt_test", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(ExitCleanly()) + Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + + check := podmanTest.Podman([]string{"inspect", "rdt_test"}) + check.WaitWithDefaultTimeout() + data := check.InspectContainerToJSON() + Expect(data[0].State.IntelRdtClosID).To(Equal("COS1")) + }) + It("podman create adds annotation", func() { session := podmanTest.Podman([]string{"create", "--annotation", "HELLO=WORLD", "--name", "annotate_test", ALPINE, "ls"}) session.WaitWithDefaultTimeout() From 77336de8f718c930cef0a8c8d31a5e70ca1c468e Mon Sep 17 00:00:00 2001 From: Wolfgang Pross Date: Tue, 26 Sep 2023 10:46:02 +0000 Subject: [PATCH 3/6] Add documentation for Intel RDT support Signed-off-by: Wolfgang Pross --- docs/source/markdown/options/rdt-class.md | 7 +++++++ docs/source/markdown/podman-create.1.md.in | 2 ++ docs/source/markdown/podman-run.1.md.in | 2 ++ 3 files changed, 11 insertions(+) create mode 100644 docs/source/markdown/options/rdt-class.md diff --git a/docs/source/markdown/options/rdt-class.md b/docs/source/markdown/options/rdt-class.md new file mode 100644 index 00000000000..7fff0e495f1 --- /dev/null +++ b/docs/source/markdown/options/rdt-class.md @@ -0,0 +1,7 @@ +####> This option file is used in: +####> podman create, run +####> If file is edited, make sure the changes +####> are applicable to all of those. +#### **--rdt-class**=*intel-rdt-class-of-service* + +Rdt-class sets the class of service (CLOS or COS) for the container to run in. Based on the Cache Allocation Technology (CAT) feature that is part of Intel's Resource Director Technology (RDT) feature set, all processes of the container will run within the pre-configured COS, that represents a part of the cache. The COS has to be created and configured by means of a pseudo file system (usually mounted at `/sys/fs/resctrl`) that the resctrl kernel driver provides. Assigning the container to a COS requires root privileges and thus doesn't work in a rootless environment. Currently the feature is only supported using `runc` as a runtime. See for more details on how to create a COS before a container can be assigned to it. diff --git a/docs/source/markdown/podman-create.1.md.in b/docs/source/markdown/podman-create.1.md.in index 0576675c5ae..0c0b325b85c 100644 --- a/docs/source/markdown/podman-create.1.md.in +++ b/docs/source/markdown/podman-create.1.md.in @@ -304,6 +304,8 @@ by having one container bind to localhost in the pod, and another connect to tha Suppress output information when pulling images +@@option rdt-class + @@option read-only @@option read-only-tmpfs diff --git a/docs/source/markdown/podman-run.1.md.in b/docs/source/markdown/podman-run.1.md.in index 018d8fe0454..76e5ea5351e 100644 --- a/docs/source/markdown/podman-run.1.md.in +++ b/docs/source/markdown/podman-run.1.md.in @@ -330,6 +330,8 @@ by having one container bind to localhost in the pod, and another connect to tha Suppress output information when pulling images +@@option rdt-class + @@option read-only @@option read-only-tmpfs From 4853320ce1bb3e89bc6ad494caf4dec4439cfb7d Mon Sep 17 00:00:00 2001 From: Wolfgang Pross Date: Tue, 26 Sep 2023 17:57:33 +0000 Subject: [PATCH 4/6] use default when user does not provide rdt-class Signed-off-by: Wolfgang Pross --- pkg/specgenutil/specgen.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index ea4fc4ca916..c15c560314b 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -492,8 +492,8 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions } // Intel RDT CAT - s.IntelRdt = &specs.LinuxIntelRdt{} if c.IntelRdtClosID != "" { + s.IntelRdt = &specs.LinuxIntelRdt{} s.IntelRdt.ClosID = c.IntelRdtClosID } From bfbd0c89609e9ed04976b6cf1fb4c4e030bcaeaa Mon Sep 17 00:00:00 2001 From: Wolfgang Pross Date: Tue, 26 Sep 2023 17:59:19 +0000 Subject: [PATCH 5/6] move IntelRdtClosID to HostConfig Signed-off-by: Wolfgang Pross --- libpod/container_inspect.go | 7 ------- libpod/container_inspect_linux.go | 8 +++++++- libpod/define/container_inspect.go | 4 +++- test/e2e/create_test.go | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 736c1914667..bb6fc7f543a 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -198,13 +198,6 @@ func (c *Container) getContainerInspectData(size bool, driverData *define.Driver } } - if ctrSpec.Linux.IntelRdt != nil { - if ctrSpec.Linux.IntelRdt.ClosID != "" { - // container is assigned to a ClosID - data.State.IntelRdtClosID = ctrSpec.Linux.IntelRdt.ClosID - } - } - networkConfig, err := c.getContainerNetworkInfo() if err != nil { return nil, err diff --git a/libpod/container_inspect_linux.go b/libpod/container_inspect_linux.go index 2dae35922f7..53fb04034bb 100644 --- a/libpod/container_inspect_linux.go +++ b/libpod/container_inspect_linux.go @@ -21,8 +21,14 @@ func (c *Container) platformInspectContainerHostConfig(ctrSpec *spec.Spec, hostC // there are things that require a major:minor to path translation. var deviceNodes map[string]string - // Resource limits if ctrSpec.Linux != nil { + if ctrSpec.Linux.IntelRdt != nil { + if ctrSpec.Linux.IntelRdt.ClosID != "" { + // container is assigned to a ClosID + hostConfig.IntelRdtClosID = ctrSpec.Linux.IntelRdt.ClosID + } + } + // Resource limits if ctrSpec.Linux.Resources != nil { if ctrSpec.Linux.Resources.CPU != nil { if ctrSpec.Linux.Resources.CPU.Shares != nil { diff --git a/libpod/define/container_inspect.go b/libpod/define/container_inspect.go index 78b3a2aedaf..879dd139595 100644 --- a/libpod/define/container_inspect.go +++ b/libpod/define/container_inspect.go @@ -229,7 +229,6 @@ type InspectContainerState struct { RestoreLog string `json:"RestoreLog,omitempty"` Restored bool `json:"Restored,omitempty"` StoppedByUser bool `json:"StoppedByUser,omitempty"` - IntelRdtClosID string `json:"IntelRdtClosID,omitempty"` } // Healthcheck returns the HealthCheckResults. This is used for old podman compat @@ -568,6 +567,9 @@ type InspectContainerHostConfig struct { IOMaximumBandwidth uint64 `json:"IOMaximumBandwidth"` // CgroupConf is the configuration for cgroup v2. CgroupConf map[string]string `json:"CgroupConf"` + // IntelRdtClosID defines the Intel RDT CAT Class Of Service (COS) that + // all processes of the container should run in. + IntelRdtClosID string `json:"IntelRdtClosID,omitempty"` } // Address represents an IP address. diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 10d0de2373a..2ec49b27945 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -84,7 +84,7 @@ var _ = Describe("Podman create", func() { check := podmanTest.Podman([]string{"inspect", "rdt_test"}) check.WaitWithDefaultTimeout() data := check.InspectContainerToJSON() - Expect(data[0].State.IntelRdtClosID).To(Equal("COS1")) + Expect(data[0].HostConfig.IntelRdtClosID).To(Equal("COS1")) }) It("podman create adds annotation", func() { From 455d165492dc5d907492433e628014d8d5b0207a Mon Sep 17 00:00:00 2001 From: wpross Date: Wed, 27 Sep 2023 16:38:16 +0200 Subject: [PATCH 6/6] Update docs/source/markdown/options/rdt-class.md Co-authored-by: Tom Sweeney Signed-off-by: wpross --- docs/source/markdown/options/rdt-class.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/markdown/options/rdt-class.md b/docs/source/markdown/options/rdt-class.md index 7fff0e495f1..8a58be528e8 100644 --- a/docs/source/markdown/options/rdt-class.md +++ b/docs/source/markdown/options/rdt-class.md @@ -4,4 +4,4 @@ ####> are applicable to all of those. #### **--rdt-class**=*intel-rdt-class-of-service* -Rdt-class sets the class of service (CLOS or COS) for the container to run in. Based on the Cache Allocation Technology (CAT) feature that is part of Intel's Resource Director Technology (RDT) feature set, all processes of the container will run within the pre-configured COS, that represents a part of the cache. The COS has to be created and configured by means of a pseudo file system (usually mounted at `/sys/fs/resctrl`) that the resctrl kernel driver provides. Assigning the container to a COS requires root privileges and thus doesn't work in a rootless environment. Currently the feature is only supported using `runc` as a runtime. See for more details on how to create a COS before a container can be assigned to it. +Rdt-class sets the class of service (CLOS or COS) for the container to run in. Based on the Cache Allocation Technology (CAT) feature that is part of Intel's Resource Director Technology (RDT) feature set, all container processes will run within the pre-configured COS, representing a part of the cache. The COS has to be created and configured using a pseudo file system (usually mounted at `/sys/fs/resctrl`) that the resctrl kernel driver provides. Assigning the container to a COS requires root privileges and thus doesn't work in a rootless environment. Currently, the feature is only supported using `runc` as a runtime. See for more details on creating a COS before a container can be assigned to it.