diff --git a/assets/performanceprofile/configs/dedicatedcpus.slice b/assets/performanceprofile/configs/dedicatedcpus.slice new file mode 100644 index 0000000000..dbad61af1a --- /dev/null +++ b/assets/performanceprofile/configs/dedicatedcpus.slice @@ -0,0 +1,5 @@ +[Unit] +Description=Top level slice for dedicated CPUs used by services/applications that requires full isolation. + +[Slice] +AllowedCPUs={{ .DedicatedCpus }} diff --git a/assets/performanceprofile/scripts/clear-irqbalance-banned-cpus.sh b/assets/performanceprofile/scripts/clear-irqbalance-banned-cpus.sh index 3ef8c61f7a..0b24e372ee 100755 --- a/assets/performanceprofile/scripts/clear-irqbalance-banned-cpus.sh +++ b/assets/performanceprofile/scripts/clear-irqbalance-banned-cpus.sh @@ -9,17 +9,51 @@ IRQBALANCE_CONF="${1:-/etc/sysconfig/irqbalance}" CRIO_ORIG_BANNED_CPUS="${2:-/etc/sysconfig/orig_irq_banned_cpus}" NONE=0 +# BANNED_CPUS: the final hex mask written to IRQBALANCE_BANNED_CPUS. +# 1. If DEDICATED_CPUS is set (via systemd Environment), use it. +# 2. Otherwise, default to 0 (no CPUs banned, all participate in balancing). +if [ -n "${DEDICATED_CPUS:-}" ]; then + BANNED_CPUS="${DEDICATED_CPUS}" +else + BANNED_CPUS="${NONE}" +fi + [ ! -f "${IRQBALANCE_CONF}" ] && exit 0 ${SED} -i '/^\s*IRQBALANCE_BANNED_CPUS\b/d' "${IRQBALANCE_CONF}" || exit 0 # CPU numbers which have their corresponding bits set to one in this mask # will not have any irq's assigned to them on rebalance. # so zero means all cpus are participating in load balancing. -echo "IRQBALANCE_BANNED_CPUS=${NONE}" >> "${IRQBALANCE_CONF}" +echo "IRQBALANCE_BANNED_CPUS=${BANNED_CPUS}" >> "${IRQBALANCE_CONF}" # we now own this configuration. But CRI-O has code to restore the configuration, # and until it gains the option to disable this restore flow, we need to make # the configuration consistent such as the CRI-O restore will do nothing. if [ -n "${CRIO_ORIG_BANNED_CPUS}" ] && [ -f "${CRIO_ORIG_BANNED_CPUS}" ]; then - echo "${NONE}" > "${CRIO_ORIG_BANNED_CPUS}" + echo "${BANNED_CPUS}" > "${CRIO_ORIG_BANNED_CPUS}" +fi + +# CRI-O reads /proc/irq/default_smp_affinity to derive the IRQ banned mask +# when pods with irq-load-balancing.crio.io=disable are scheduled. +# If we don't remove the dedicated CPUs from default_smp_affinity here, +# CRI-O will overwrite IRQBALANCE_BANNED_CPUS while ignoring the dedicated CPUs. +SMP_AFFINITY="/proc/irq/default_smp_affinity" +if [ "${BANNED_CPUS}" != "${NONE}" ] && [ -f "${SMP_AFFINITY}" ]; then + # default_smp_affinity is comma-separated 32-bit hex groups (e.g. "ff,ffffffff,ffffffff") + IFS=',' read -ra smp < "${SMP_AFFINITY}" + n=${#smp[@]} + # pad BANNED_CPUS with leading zeros to match the same number of hex chars + padded="${BANNED_CPUS}" + while [ ${#padded} -lt $(( n * 8 )) ]; do + padded="0${padded}" + done + # clear banned bits from each 32-bit group: result = smp & ~banned + result="" + for (( i=0; i "${SMP_AFFINITY}" fi diff --git a/assets/performanceprofile/scripts/dedicated-cpus-configure.sh b/assets/performanceprofile/scripts/dedicated-cpus-configure.sh new file mode 100644 index 0000000000..cd066e6e0f --- /dev/null +++ b/assets/performanceprofile/scripts/dedicated-cpus-configure.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# dedicated-cpus-configure.sh configures the dedicatedcpus.slice cpuset +# partition so that the dedicated CPUs are fully isolated from the kernel +# scheduler (equivalent to isolcpus=domain for those specific CPUs). + +set -euo pipefail + +DEDICATED_CPUS="{{ .DedicatedCpus }}" + +if [ -z "$DEDICATED_CPUS" ]; then + echo "No dedicated CPUs configured, nothing to do" + exit 0 +fi + +CGROUP_PATH="/sys/fs/cgroup/dedicatedcpus.slice" + +if [ ! -d "$CGROUP_PATH" ]; then + echo "ERROR: dedicatedcpus.slice cgroup does not exist at $CGROUP_PATH" >&2 + exit 1 +fi + +# Set exclusive CPUs on the dedicated slice +echo "$DEDICATED_CPUS" > "$CGROUP_PATH/cpuset.cpus.exclusive" + +# Set the CPUs available to the slice +echo "$DEDICATED_CPUS" > "$CGROUP_PATH/cpuset.cpus" + +# Create an isolated partition — removes these CPUs from the parent's +# scheduling domain, giving kernel-level isolation equivalent to +# isolcpus=domain without affecting other CPUs. +echo "isolated" > "$CGROUP_PATH/cpuset.cpus.partition" + +echo "Configured dedicatedcpus.slice as isolated partition for CPUs: $DEDICATED_CPUS" diff --git a/manifests/20-performance-profile.crd.yaml b/manifests/20-performance-profile.crd.yaml index 9bed853528..98277d8bbe 100644 --- a/manifests/20-performance-profile.crd.yaml +++ b/manifests/20-performance-profile.crd.yaml @@ -509,6 +509,16 @@ spec: offloads the complexity of cpu load balancing to the application. Defaults to "true" type: boolean + dedicated: + description: |- + Dedicated defines a set of CPUs fully isolated from the operating system + and Kubernetes scheduling, intended for exclusive use by user-space + processes (for example, infrastructure networking workloads such as + DPDK-based vSwitch or vRouter). These CPUs receive full kernel-level + isolation (isolcpus=domain,managed_irq, nohz_full, rcu_nocbs), are + excluded from Kubelet scheduling (all QoS classes), banned from + irqbalance, and excluded from systemd CPU affinity. + type: string isolated: description: |- Isolated defines a set of CPUs that will be used to give to application threads the most execution time possible, @@ -625,6 +635,11 @@ spec: vendorID: description: Network device vendor ID represnted as a 16 bit Hexmadecimal number. type: string + disableOvsDynamicPinning: + description: |- + DisableOvsDynamicPinning when set to true, prevents OVN-Kubernetes + from dynamically adjusting OVS thread CPU affinity at runtime. + type: boolean userLevelNetworking: description: UserLevelNetworking when enabled - sets either all or specified network devices queue size to the amount of reserved CPUs. Defaults to "false". type: boolean diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_types.go b/pkg/apis/performanceprofile/v2/performanceprofile_types.go index f877fcdcdb..859c9c59a1 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_types.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_types.go @@ -138,6 +138,13 @@ type CPU struct { // alongside the isolated, exclusive resources that are being used already by those workloads. // +optional Shared *CPUSet `json:"shared,omitempty"` + // Dedicated defines a set of CPUs fully isolated from the operating system + // and Kubernetes scheduling, intended for exclusive use by user-space + // processes (for example, infrastructure networking workloads such as + // DPDK-based vSwitch or vRouter). WorkloadPartitioning or --strict-cpu-reservation + // kubelet CPUManager policy option are a prerequisite for this feature. + // +optional + Dedicated *CPUSet `json:"dedicated,omitempty"` } // CPUfrequency defines cpu frequencies for isolated and reserved cpus @@ -203,6 +210,10 @@ type Net struct { // set with a netqueue count equal to CPU.Reserved . // If no devices are specified then the default is all devices. Devices []Device `json:"devices,omitempty"` + // DisableOvsDynamicPinning when set to true, prevents OVN-Kubernetes + // from dynamically adjusting OVS thread CPU affinity at runtime. + // +optional + DisableOvsDynamicPinning *bool `json:"disableOvsDynamicPinning,omitempty"` } // Device defines a way to represent a network device in several options: diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_validation.go b/pkg/apis/performanceprofile/v2/performanceprofile_validation.go index 7da8547f70..fa6305beca 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_validation.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_validation.go @@ -187,14 +187,17 @@ func (r *PerformanceProfile) validateCPUs() field.ErrorList { } if cpus.Isolated != nil && cpus.Reserved != nil { - var offlined, shared string + var offlined, shared, dedicated string if cpus.Offlined != nil { offlined = string(*cpus.Offlined) } if cpus.Shared != nil { shared = string(*cpus.Shared) } - cpuLists, err := components.NewCPULists(string(*cpus.Reserved), string(*cpus.Isolated), offlined, shared) + if cpus.Dedicated != nil { + dedicated = string(*cpus.Dedicated) + } + cpuLists, err := components.NewCPULists(string(*cpus.Reserved), string(*cpus.Isolated), offlined, shared, dedicated) if err != nil { allErrs = append(allErrs, field.InternalError(field.NewPath("spec.cpu"), err)) // If err != nil then the cpuList is nil and we can't continue with the function logic diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_validation_test.go b/pkg/apis/performanceprofile/v2/performanceprofile_validation_test.go index cc4e99a713..8951ed7aee 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_validation_test.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_validation_test.go @@ -165,10 +165,11 @@ func FuzzValidateCPUs(f *testing.F) { } f.Fuzz(func(t *testing.T, input string) { cpuFields := map[string]func(*PerformanceProfile, CPUSet){ - "reserved": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Reserved = &input }, - "isolated": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Isolated = &input }, - "shared": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Shared = &input }, - "offline": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Offlined = &input }, + "reserved": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Reserved = &input }, + "isolated": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Isolated = &input }, + "shared": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Shared = &input }, + "offline": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Offlined = &input }, + "dedicated": func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Dedicated = &input }, } for fieldName, setField := range cpuFields { @@ -296,6 +297,74 @@ var _ = Describe("PerformanceProfile", func() { Expect(errors).NotTo(BeEmpty(), "should have validation error when isolated and shared CPUs have overlap") Expect(errors[0].Error()).To(Or(ContainSubstring("isolated and shared cpus overlap"), ContainSubstring("shared and isolated cpus overlap"))) }) + + It("should allow valid dedicated CPUs that do not overlap with other sets", func() { + reservedCPUs := CPUSet("0-1") + isolatedCPUs := CPUSet("4-7") + dedicatedCPUs := CPUSet("2-3") + profile.Spec.CPU.Reserved = &reservedCPUs + profile.Spec.CPU.Isolated = &isolatedCPUs + profile.Spec.CPU.Offlined = nil + profile.Spec.CPU.Dedicated = &dedicatedCPUs + errors := profile.validateCPUs() + Expect(errors).To(BeEmpty(), "should not have validation errors with non-overlapping dedicated CPUs") + }) + + It("should reject cpus allocation with overlapping sets between dedicated and reserved", func() { + reservedCPUs := CPUSet("0-3") + isolatedCPUs := CPUSet("8-11") + dedicatedCPUs := CPUSet("2-5") + profile.Spec.CPU.Reserved = &reservedCPUs + profile.Spec.CPU.Isolated = &isolatedCPUs + profile.Spec.CPU.Offlined = nil + profile.Spec.CPU.Dedicated = &dedicatedCPUs + errors := profile.validateCPUs() + Expect(errors).NotTo(BeEmpty(), "should have validation error when dedicated and reserved CPUs have overlap") + Expect(errors[0].Error()).To(Or(ContainSubstring("dedicated and reserved cpus overlap"), ContainSubstring("reserved and dedicated cpus overlap"))) + }) + + It("should reject cpus allocation with overlapping sets between dedicated and isolated", func() { + reservedCPUs := CPUSet("0-1") + isolatedCPUs := CPUSet("4-7") + dedicatedCPUs := CPUSet("6-9") + profile.Spec.CPU.Reserved = &reservedCPUs + profile.Spec.CPU.Isolated = &isolatedCPUs + profile.Spec.CPU.Offlined = nil + profile.Spec.CPU.Dedicated = &dedicatedCPUs + errors := profile.validateCPUs() + Expect(errors).NotTo(BeEmpty(), "should have validation error when dedicated and isolated CPUs have overlap") + Expect(errors[0].Error()).To(Or(ContainSubstring("dedicated and isolated cpus overlap"), ContainSubstring("isolated and dedicated cpus overlap"))) + }) + + It("should reject cpus allocation with overlapping sets between dedicated and offlined", func() { + reservedCPUs := CPUSet("0-1") + isolatedCPUs := CPUSet("4-5") + offlinedCPUs := CPUSet("6-7") + dedicatedCPUs := CPUSet("2-3,7") + profile.Spec.CPU.Reserved = &reservedCPUs + profile.Spec.CPU.Isolated = &isolatedCPUs + profile.Spec.CPU.Offlined = &offlinedCPUs + profile.Spec.CPU.Dedicated = &dedicatedCPUs + errors := profile.validateCPUs() + Expect(errors).NotTo(BeEmpty(), "should have validation error when dedicated and offlined CPUs have overlap") + Expect(errors[0].Error()).To(Or(ContainSubstring("dedicated and offlined cpus overlap"), ContainSubstring("offlined and dedicated cpus overlap"))) + }) + + It("should reject cpus allocation with overlapping sets between dedicated and shared", func() { + reservedCPUs := CPUSet("0-1") + isolatedCPUs := CPUSet("4-5") + sharedCPUs := CPUSet("6-7") + dedicatedCPUs := CPUSet("2-3,6") + profile.Spec.CPU.Reserved = &reservedCPUs + profile.Spec.CPU.Isolated = &isolatedCPUs + profile.Spec.CPU.Offlined = nil + profile.Spec.CPU.Shared = &sharedCPUs + profile.Spec.CPU.Dedicated = &dedicatedCPUs + errors := profile.validateCPUs() + Expect(errors).NotTo(BeEmpty(), "should have validation error when dedicated and shared CPUs have overlap") + Expect(errors[0].Error()).To(Or(ContainSubstring("dedicated and shared cpus overlap"), ContainSubstring("shared and dedicated cpus overlap"))) + }) + DescribeTable("should reject invalid input that does not represent CPU sets", func(fieldSetter func(*PerformanceProfile, CPUSet), cpusField string) { garbageInput := CPUSet("garbage") @@ -308,6 +377,7 @@ var _ = Describe("PerformanceProfile", func() { Entry("isolated CPUs", func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Isolated = &input }, "isolated CPUs"), Entry("shared CPUs", func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Shared = &input }, "shared CPUs"), Entry("offline CPUs", func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Offlined = &input }, "offline CPUs"), + Entry("dedicated CPUs", func(p *PerformanceProfile, input CPUSet) { p.Spec.CPU.Dedicated = &input }, "dedicated CPUs"), ) }) diff --git a/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go b/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go index 95af183562..a1ec6451d0 100644 --- a/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go +++ b/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go @@ -53,6 +53,11 @@ func (in *CPU) DeepCopyInto(out *CPU) { *out = new(CPUSet) **out = **in } + if in.Dedicated != nil { + in, out := &in.Dedicated, &out.Dedicated + *out = new(CPUSet) + **out = **in + } return } @@ -208,6 +213,11 @@ func (in *Net) DeepCopyInto(out *Net) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.DisableOvsDynamicPinning != nil { + in, out := &in.DisableOvsDynamicPinning, &out.DisableOvsDynamicPinning + *out = new(bool) + **out = **in + } return } diff --git a/pkg/performanceprofile/controller/performanceprofile/components/components.go b/pkg/performanceprofile/controller/performanceprofile/components/components.go index 0ec412e998..4fcc013f8a 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/components.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/components.go @@ -42,8 +42,10 @@ type Options struct { } type MachineConfigOptions struct { - PinningMode *apiconfigv1.CPUPartitioningMode - MixedCPUsEnabled bool + PinningMode *apiconfigv1.CPUPartitioningMode + MixedCPUsEnabled bool + DisableOVSDynamicPinning bool + DedicatedCPUs string } type KubeletConfigOptions struct { diff --git a/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go b/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go index a542880620..90e65117d4 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/handler/handler.go @@ -43,14 +43,18 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record. klog.Infof("Ignoring reconcile loop for pause performance profile %s", profile.Name) return nil } - // set missing options - opts.MachineConfig.MixedCPUsEnabled = opts.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile) - opts.DRAResourceManagement = profileutil.IsDRAManaged(profile) + + profileutil.SetMissingOptions(profile, opts) components, err := manifestset.GetNewComponents(profile, opts) if err != nil { return err } + + if err := profileutil.ValidateDedicatedCPUsPrerequisites(profile, opts, components.KubeletConfig); err != nil { + return err + } + for _, componentObj := range components.ToObjects() { if err := controllerutil.SetControllerReference(profile, componentObj, h.scheme); err != nil { return err diff --git a/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go b/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go index a78be62ae3..6a17f8b523 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig.go @@ -9,6 +9,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" "k8s.io/utils/cpuset" @@ -31,6 +32,7 @@ const ( experimentalKubeletSnippetAnnotation = "kubeletconfig.experimental" cpuManagerPolicyStatic = "static" cpuManagerPolicyOptionFullPCPUsOnly = "full-pcpus-only" + cpuManagerPolicyOptionStrictCPUReservation = "strict-cpu-reservation" memoryManagerPolicyStatic = "Static" defaultKubeReservedMemory = "500Mi" defaultSystemReservedMemory = "500Mi" @@ -156,19 +158,29 @@ func setKubeletConfigForCPUAndMemoryManagers(profile *performancev2.PerformanceP } if profile.Spec.CPU != nil && profile.Spec.CPU.Reserved != nil { - kubeletConfig.ReservedSystemCPUs = string(*profile.Spec.CPU.Reserved) - } - - if opts.MixedCPUsEnabled { - sharedCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Shared)) - if err != nil { - return err - } reservedCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Reserved)) if err != nil { return err } - kubeletConfig.ReservedSystemCPUs = reservedCPUs.Union(sharedCPUs).String() + + if profile.Spec.CPU.Dedicated != nil { + dedicatedCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Dedicated)) + if err != nil { + return err + } + // dedicated CPUs are added to the reserved CPUs set to ensure they are not used for any container workloads initiated by kubelet. + reservedCPUs = reservedCPUs.Union(dedicatedCPUs) + } + + if opts.MixedCPUsEnabled { + sharedCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Shared)) + if err != nil { + return err + } + reservedCPUs = reservedCPUs.Union(sharedCPUs) + } + + kubeletConfig.ReservedSystemCPUs = reservedCPUs.String() } if profile.Spec.NUMA != nil { @@ -232,3 +244,20 @@ func validateOptions(opts *components.KubeletConfigOptions) error { return nil } + +// HasStrictCPUReservation returns true if the generated KubeletConfig has the +// strict-cpu-reservation CPUManager policy option enabled. +func HasStrictCPUReservation(kc *machineconfigv1.KubeletConfig) bool { + if kc == nil || kc.Spec.KubeletConfig == nil || len(kc.Spec.KubeletConfig.Raw) == 0 { + return false + } + + kubeletConfig := &kubeletconfigv1beta1.KubeletConfiguration{} + if err := json.Unmarshal(kc.Spec.KubeletConfig.Raw, kubeletConfig); err != nil { + klog.V(4).Infof("failed to unmarshal KubeletConfig payload: %v", err) + return false + } + + v, ok := kubeletConfig.CPUManagerPolicyOptions[cpuManagerPolicyOptionStrictCPUReservation] + return ok && v == "true" +} diff --git a/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go b/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go index 3646034123..a00226f3c0 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig/kubeletconfig_test.go @@ -9,6 +9,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/yaml" + performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" k8seviction "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/k8simported/eviction" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" @@ -207,6 +208,55 @@ var _ = Describe("Kubelet Config", func() { }) + Context("with dedicated CPUs", func() { + It("should include dedicated CPUs in reservedSystemCPUs", func() { + profile := testutils.NewPerformanceProfile("test") + dedicatedCPUs := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicatedCPUs + selectorKey, selectorValue := components.GetFirstKeyAndValue(profile.Spec.MachineConfigPoolSelector) + kc, err := New(profile, &components.KubeletConfigOptions{MachineConfigPoolSelector: map[string]string{selectorKey: selectorValue}}) + Expect(err).ToNot(HaveOccurred()) + + y, err := yaml.Marshal(kc) + Expect(err).ToNot(HaveOccurred()) + + manifest := string(y) + Expect(manifest).To(ContainSubstring("reservedSystemCPUs: 0-3,10-11")) + }) + + It("should include both dedicated and shared CPUs in reservedSystemCPUs when mixed CPUs is enabled", func() { + profile := testutils.NewPerformanceProfile("test") + dedicatedCPUs := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicatedCPUs + selectorKey, selectorValue := components.GetFirstKeyAndValue(profile.Spec.MachineConfigPoolSelector) + kc, err := New(profile, &components.KubeletConfigOptions{ + MachineConfigPoolSelector: map[string]string{selectorKey: selectorValue}, + MixedCPUsEnabled: true, + }) + Expect(err).ToNot(HaveOccurred()) + + y, err := yaml.Marshal(kc) + Expect(err).ToNot(HaveOccurred()) + + manifest := string(y) + Expect(manifest).To(ContainSubstring("reservedSystemCPUs: 0-3,8-11")) + }) + + It("should not change reservedSystemCPUs when dedicated is nil", func() { + profile := testutils.NewPerformanceProfile("test") + profile.Spec.CPU.Dedicated = nil + selectorKey, selectorValue := components.GetFirstKeyAndValue(profile.Spec.MachineConfigPoolSelector) + kc, err := New(profile, &components.KubeletConfigOptions{MachineConfigPoolSelector: map[string]string{selectorKey: selectorValue}}) + Expect(err).ToNot(HaveOccurred()) + + y, err := yaml.Marshal(kc) + Expect(err).ToNot(HaveOccurred()) + + manifest := string(y) + Expect(manifest).To(ContainSubstring("reservedSystemCPUs: 0-3")) + }) + }) + Context("with mutually exclusive options", func() { It("should return an error when both MixedCPUs and DRA resource management are enabled", func() { profile := testutils.NewPerformanceProfile("test") diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go index aa96600de1..bcd2aca93a 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig.go @@ -75,6 +75,10 @@ const ( ovsDynamicPinningTriggerFile = "ovs-enable-dynamic-cpu-affinity" ovsDynamicPinningTriggerHostFile = "/var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity" + dedicatedCPUsSliceName = "dedicatedcpus.slice" + dedicatedCPUsSliceDefinitionFile = "dedicatedcpus.slice" + dedicatedCPUsConfigure = "dedicated-cpus-configure" + cpusetConfigure = "cpuset-configure" // ExecCPUAffinity config @@ -123,6 +127,7 @@ const ( templateCrioSharedCPUsAnnotation = "CrioSharedCPUsAnnotation" templateExecCPUAffinity = "ExecCPUAffinity" templateMinInjectedGOMAXPROCS = "MinInjectedGOMAXPROCS" + templateDedicatedCpus = "DedicatedCpus" ) const crioMinGOMAXPROCS = 4 @@ -339,7 +344,7 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile, opts *componen }) } - clearIRQBalanceBannedCPUsService, err := getSystemdContent(getIRQBalanceBannedCPUsOptions()) + clearIRQBalanceBannedCPUsService, err := getSystemdContent(getIRQBalanceBannedCPUsOptions(opts.DedicatedCPUs)) if err != nil { return nil, err } @@ -386,12 +391,45 @@ func getIgnitionConfig(profile *performancev2.PerformanceProfile, opts *componen addContent(ignitionConfig, serviceOvsSlice, "/etc/systemd/system/ovs-vswitchd.service.d/"+templateOvsSliceUsageFile, &ovsMode) addContent(ignitionConfig, serviceOvsSlice, "/etc/systemd/system/ovsdb-server.service.d/"+templateOvsSliceUsageFile, &ovsMode) - // Tell OVN-K to enable dynamic cpu pinning - content, err := getTemplatedOvsFile(assets.Configs, filepath.Join("configs", ovsDynamicPinningTriggerFile), ovsSliceName) + if !opts.DisableOVSDynamicPinning { + content, err := getTemplatedOvsFile(assets.Configs, filepath.Join("configs", ovsDynamicPinningTriggerFile), ovsSliceName) + if err != nil { + return nil, err + } + addContent(ignitionConfig, content, ovsDynamicPinningTriggerHostFile, &ovsMode) + } + } + + if opts.DedicatedCPUs != "" { + sliceContent, err := renderTemplatedFile(assets.Configs, filepath.Join("configs", dedicatedCPUsSliceDefinitionFile), map[string]string{ + templateDedicatedCpus: opts.DedicatedCPUs, + }) if err != nil { return nil, err } - addContent(ignitionConfig, content, ovsDynamicPinningTriggerHostFile, &ovsMode) + dedicatedMode := 0644 + addContent(ignitionConfig, sliceContent, "/etc/systemd/system/"+dedicatedCPUsSliceName, &dedicatedMode) + + scriptContent, err := renderTemplatedFile(assets.Scripts, fmt.Sprintf("scripts/%s.sh", dedicatedCPUsConfigure), map[string]string{ + templateDedicatedCpus: opts.DedicatedCPUs, + }) + if err != nil { + return nil, err + } + dst := getBashScriptPath(dedicatedCPUsConfigure) + scriptFileMode := 0700 + addContent(ignitionConfig, scriptContent, dst, &scriptFileMode) + + dedicatedConfigureServiceContent, err := getSystemdContent(getDedicatedCpusConfigureOptions()) + if err != nil { + return nil, err + } + dedicatedConfigureService := dedicatedConfigureServiceContent + ignitionConfig.Systemd.Units = append(ignitionConfig.Systemd.Units, igntypes.Unit{ + Contents: &dedicatedConfigureService, + Enabled: ptr.To(true), + Name: getSystemdService(dedicatedCPUsConfigure), + }) } // Configure a systemd dropin and sysconfig file so stalld uses sched_debug as its backend. @@ -458,21 +496,22 @@ func GetHugepagesSizeKilobytes(hugepagesSize performancev2.HugePageSize) (string return strconv.FormatInt(size/1024, 10), nil } -func getTemplatedOvsFile(fsys fs.FS, templateName string, name string) ([]byte, error) { - templateArgs := make(map[string]string) - templateArgs[templateOvsSliceName] = name - - sliceTemplate, err := template.ParseFS(fsys, templateName) +func renderTemplatedFile(fsys fs.FS, templateName string, args map[string]string) ([]byte, error) { + tmpl, err := template.ParseFS(fsys, templateName) if err != nil { return nil, err } - - slice := &bytes.Buffer{} - if err := sliceTemplate.Execute(slice, templateArgs); err != nil { + var buf bytes.Buffer + if err := tmpl.Execute(&buf, args); err != nil { return nil, err } + return buf.Bytes(), nil +} - return slice.Bytes(), nil +func getTemplatedOvsFile(fsys fs.FS, templateName string, name string) ([]byte, error) { + return renderTemplatedFile(fsys, templateName, map[string]string{ + templateOvsSliceName: name, + }) } func getOvsSliceDefinition(name string) ([]byte, error) { @@ -502,8 +541,32 @@ func getCpusetConfigureServiceOptions() []*unit.UnitOption { } } -func getIRQBalanceBannedCPUsOptions() []*unit.UnitOption { +func getDedicatedCpusConfigureOptions() []*unit.UnitOption { return []*unit.UnitOption{ + // [Unit] + // Description + unit.NewUnitOption(systemdSectionUnit, systemdDescription, "Configure cgroup v2 cpuset partition for dedicated CPUs"), + // Requires + unit.NewUnitOption(systemdSectionUnit, "Requires", dedicatedCPUsSliceName), + // Before + unit.NewUnitOption(systemdSectionUnit, systemdBefore, systemdServiceKubelet), + // After + unit.NewUnitOption(systemdSectionUnit, systemdAfter, dedicatedCPUsSliceName), + // [Service] + // Type + unit.NewUnitOption(systemdSectionService, systemdType, systemdServiceTypeOneshot), + // RemainAfterExit + unit.NewUnitOption(systemdSectionService, systemdRemainAfterExit, systemdTrue), + // ExecStart + unit.NewUnitOption(systemdSectionService, systemdExecStart, getBashScriptPath(dedicatedCPUsConfigure)), + // [Install] + // WantedBy + unit.NewUnitOption(systemdSectionInstall, systemdWantedBy, systemdTargetMultiUser), + } +} + +func getIRQBalanceBannedCPUsOptions(dedicatedCPUs string) []*unit.UnitOption { + opts := []*unit.UnitOption{ // [Unit] // Description unit.NewUnitOption(systemdSectionUnit, systemdDescription, "Clear the IRQBalance Banned CPU mask early in the boot"), @@ -521,6 +584,17 @@ func getIRQBalanceBannedCPUsOptions() []*unit.UnitOption { // WantedBy unit.NewUnitOption(systemdSectionInstall, systemdWantedBy, systemdTargetMultiUser), } + if dedicatedCPUs != "" { + dedicatedMask, err := components.CPUListToHexMask(dedicatedCPUs) + if err != nil { + klog.Errorf("failed to convert dedicated CPUs %q to hex mask: %v", dedicatedCPUs, err) + return opts + } + opts = append(opts, + unit.NewUnitOption(systemdSectionService, systemdEnvironment, getSystemdEnvironment("DEDICATED_CPUS", dedicatedMask)), + ) + } + return opts } func getHugepagesAllocationUnitOptions(hugepagesSize string, hugepagesCount int32, numaNode int32) []*unit.UnitOption { diff --git a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go index d6d2372d19..67d02de4aa 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/machineconfig/machineconfig_test.go @@ -314,7 +314,7 @@ var _ = Describe("Machine Config", func() { Context("check systemd units", func() { It("should generate clear-banned-cpus unit", func() { - unit, err := getSystemdContent(getIRQBalanceBannedCPUsOptions()) + unit, err := getSystemdContent(getIRQBalanceBannedCPUsOptions("")) Expect(err).ToNot(HaveOccurred()) expected := `[Unit] Description=Clear the IRQBalance Banned CPU mask early in the boot @@ -468,6 +468,197 @@ resources = { "cpushares" = 0, "cpuset" = "" } }) }) +var _ = Describe("Machine Config dedicated CPUs and OVS dynamic pinning", func() { + Context("OVS dynamic pinning trigger file", func() { + It("should be present when DisableOvsDynamicPinning is false", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DisableOVSDynamicPinning: false, + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + found := false + for _, f := range result.Storage.Files { + if f.Path == ovsDynamicPinningTriggerHostFile { + found = true + break + } + } + Expect(found).To(BeTrue(), "OVS dynamic pinning trigger file should be present") + }) + + It("should be absent when DisableOvsDynamicPinning is true", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DisableOVSDynamicPinning: true, + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + for _, f := range result.Storage.Files { + Expect(f.Path).ToNot(Equal(ovsDynamicPinningTriggerHostFile), + "OVS dynamic pinning trigger file should not be present") + } + }) + + It("should still create OVS slice when DisableOvsDynamicPinning is true", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DisableOVSDynamicPinning: true, + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + foundOvsSlice := false + for _, f := range result.Storage.Files { + if strings.Contains(f.Path, ovsSliceName) { + foundOvsSlice = true + break + } + } + Expect(foundOvsSlice).To(BeTrue(), "OVS slice definition should still be present") + }) + }) + + Context("dedicated CPUs slice and configure service", func() { + It("should create dedicatedcpus.slice when dedicated CPUs are specified", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DedicatedCPUs: "4-7", + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + found := false + for _, f := range result.Storage.Files { + if f.Path == "/etc/systemd/system/dedicatedcpus.slice" { + found = true + break + } + } + Expect(found).To(BeTrue(), "dedicatedcpus.slice should be present in ignition files") + }) + + It("should create dedicated-cpus-configure service when dedicated CPUs are specified", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DedicatedCPUs: "4-7", + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + found := false + for _, u := range result.Systemd.Units { + if u.Name == "dedicated-cpus-configure.service" { + found = true + Expect(u.Enabled).ToNot(BeNil()) + Expect(*u.Enabled).To(BeTrue()) + } + } + Expect(found).To(BeTrue(), "dedicated-cpus-configure.service should be present") + }) + + It("should create dedicated-cpus-configure script when dedicated CPUs are specified", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{ + DedicatedCPUs: "4-7", + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + found := false + for _, f := range result.Storage.Files { + if f.Path == "/usr/local/bin/dedicated-cpus-configure.sh" { + found = true + break + } + } + Expect(found).To(BeTrue(), "dedicated-cpus-configure.sh script should be present") + }) + + It("should not create dedicatedcpus.slice when dedicated CPUs are not specified", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{}) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + for _, f := range result.Storage.Files { + Expect(f.Path).ToNot(Equal("/etc/systemd/system/dedicatedcpus.slice"), + "dedicatedcpus.slice should not be present when no dedicated CPUs") + } + for _, u := range result.Systemd.Units { + Expect(u.Name).ToNot(Equal("dedicated-cpus-configure.service"), + "dedicated-cpus-configure.service should not be present when no dedicated CPUs") + } + }) + }) + + Context("IRQBALANCE_BANNED_CPUS for dedicated CPUs", func() { + It("should set IRQBALANCE_BANNED_CPUS when dedicated CPUs are specified", func() { + profile := testutils.NewPerformanceProfile("test") + dedicatedCPUs := performancev2.CPUSet("2-3") + profile.Spec.CPU.Dedicated = &dedicatedCPUs + mc, err := New(profile, &components.MachineConfigOptions{ + DedicatedCPUs: "2-3", + }) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + found := false + for _, u := range result.Systemd.Units { + if u.Name == "clear-irqbalance-banned-cpus.service" && u.Contents != nil { + if strings.Contains(*u.Contents, "DEDICATED_CPUS=c") { + found = true + } + } + } + Expect(found).To(BeTrue(), "DEDICATED_CPUS should be set to hex mask of dedicated CPUs 2-3 (0xc)") + }) + + It("should not set IRQBALANCE_BANNED_CPUS when dedicated CPUs are not specified", func() { + profile := testutils.NewPerformanceProfile("test") + mc, err := New(profile, &components.MachineConfigOptions{}) + Expect(err).ToNot(HaveOccurred()) + + result := igntypes.Config{} + err = json.Unmarshal(mc.Spec.Config.Raw, &result) + Expect(err).ToNot(HaveOccurred()) + + for _, u := range result.Systemd.Units { + if u.Name == "clear-irqbalance-banned-cpus.service" && u.Contents != nil { + Expect(*u.Contents).ToNot(ContainSubstring("IRQBALANCE_BANNED_CPUS"), + "IRQBALANCE_BANNED_CPUS should not be set when no dedicated CPUs") + } + } + }) + }) +}) + func removeAllWhiteSpace(str string) string { return spaceRegex.ReplaceAllString(str, "") } diff --git a/pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go b/pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go index 7026bd1f14..2dc75ec1a9 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/profile/profile.go @@ -5,10 +5,12 @@ import ( "k8s.io/klog/v2" + apiconfigv1 "github.com/openshift/api/config/v1" + mcov1 "github.com/openshift/api/machineconfiguration/v1" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" - - mcov1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components/kubeletconfig" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/status" ) // GetMachineConfigPoolSelector returns the MachineConfigPoolSelector from the CR or a default value calculated based on NodeSelector @@ -130,3 +132,32 @@ func IsDRAManaged(profile *performancev2.PerformanceProfile) bool { } return parsed } + +// ValidateDedicatedCPUsPrerequisites checks that if dedicated CPUs are set, +// either Workload Partitioning or strict-cpu-reservation is enabled. +func ValidateDedicatedCPUsPrerequisites(profile *performancev2.PerformanceProfile, opts *components.Options, kc *mcov1.KubeletConfig) error { + if profile.Spec.CPU == nil || profile.Spec.CPU.Dedicated == nil { + return nil + } + if opts.MachineConfig.PinningMode != nil && *opts.MachineConfig.PinningMode == apiconfigv1.CPUPartitioningAllNodes { + return nil + } + if kubeletconfig.HasStrictCPUReservation(kc) { + return nil + } + return status.NewDedicatedCPUsPrerequisiteError() +} + +// SetMissingOptions populates derived Options fields from the profile spec. +func SetMissingOptions(profile *performancev2.PerformanceProfile, opts *components.Options) { + opts.MachineConfig.MixedCPUsEnabled = opts.MixedCPUsFeatureGateEnabled && IsMixedCPUsEnabled(profile) + opts.DRAResourceManagement = IsDRAManaged(profile) + + if profile.Spec.Net != nil && profile.Spec.Net.DisableOvsDynamicPinning != nil { + opts.MachineConfig.DisableOVSDynamicPinning = *profile.Spec.Net.DisableOvsDynamicPinning + } + + if profile.Spec.CPU != nil && profile.Spec.CPU.Dedicated != nil { + opts.MachineConfig.DedicatedCPUs = string(*profile.Spec.CPU.Dedicated) + } +} diff --git a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go index c15f4a01f0..a6b904860d 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go @@ -63,12 +63,21 @@ func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tun templateArgs[templatePerformanceProfileName] = profile.Name if profile.Spec.CPU.Isolated != nil { - minifiedCpuSet, err := cpuset.Parse(string(*profile.Spec.CPU.Isolated)) + isolatedSet, err := cpuset.Parse(string(*profile.Spec.CPU.Isolated)) if err != nil { return nil, fmt.Errorf("cannot parse isolated cpuset: %v", err) } - templateArgs[templateIsolatedCpus] = minifiedCpuSet.String() - templateArgs[templateIsolatedCpuList] = minifiedCpuSet.List() + + if profile.Spec.CPU.Dedicated != nil { + dedicatedSet, err := cpuset.Parse(string(*profile.Spec.CPU.Dedicated)) + if err != nil { + return nil, fmt.Errorf("cannot parse dedicated cpuset: %v", err) + } + isolatedSet = isolatedSet.Union(dedicatedSet) + } + + templateArgs[templateIsolatedCpus] = isolatedSet.String() + templateArgs[templateIsolatedCpuList] = isolatedSet.List() } if profile.Spec.CPU.Reserved != nil { diff --git a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go index 592f7dec86..01f0ac73a4 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned_test.go @@ -727,4 +727,27 @@ var _ = Describe("Tuned", func() { }) }) }) + + Context("with dedicated CPUs", func() { + It("should include dedicated CPUs in isolated_cores", func() { + dedicatedCPUs := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicatedCPUs + tunedData := getTunedStructuredData(profile, components.ProfileNamePerformance) + variables, err := tunedData.GetSection("variables") + Expect(err).ToNot(HaveOccurred()) + isolatedCores := variables.Key("isolated_cores").String() + set, err := cpuset.Parse(isolatedCores) + Expect(err).ToNot(HaveOccurred()) + Expect(set.List()).To(ContainElements(4, 5, 10, 11)) + }) + + It("should include dedicated CPUs in nohz_full and rcu_nocbs via isolated_cores", func() { + dedicatedCPUs := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicatedCPUs + tunedData := getTunedStructuredData(profile, components.ProfileNamePerformance) + bootLoaderSection, err := tunedData.GetSection("bootloader") + Expect(err).ToNot(HaveOccurred()) + Expect(bootLoaderSection.Key("cmdline_cpu_part").String()).To(Equal(cmdlineCPUsPartitioning)) + }) + }) }) diff --git a/pkg/performanceprofile/controller/performanceprofile/components/utils.go b/pkg/performanceprofile/controller/performanceprofile/components/utils.go index 884c3c5f93..05747dacbf 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/utils.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/utils.go @@ -110,12 +110,16 @@ func (c *CPULists) GetShared() cpuset.CPUSet { return c.sets["shared"] } +func (c *CPULists) GetDedicated() cpuset.CPUSet { + return c.sets["dedicated"] +} + func (c *CPULists) GetSets() map[string]cpuset.CPUSet { return c.sets } // NewCPULists parse text representations of reserved and isolated cpusets definition and returns a CPULists object -func NewCPULists(reserved, isolated, offlined, shared string) (*CPULists, error) { +func NewCPULists(reserved, isolated, offlined, shared, dedicated string) (*CPULists, error) { reservedSet, err := cpuset.Parse(reserved) if err != nil { return nil, err @@ -132,12 +136,17 @@ func NewCPULists(reserved, isolated, offlined, shared string) (*CPULists, error) if err != nil { return nil, err } + dedicatedSet, err := cpuset.Parse(dedicated) + if err != nil { + return nil, err + } return &CPULists{ sets: map[string]cpuset.CPUSet{ - "reserved": reservedSet, - "isolated": isolatedSet, - "offlined": offlinedSet, - "shared": sharedSet, + "reserved": reservedSet, + "isolated": isolatedSet, + "offlined": offlinedSet, + "shared": sharedSet, + "dedicated": dedicatedSet, }, }, nil } diff --git a/pkg/performanceprofile/controller/performanceprofile/components/utils_test.go b/pkg/performanceprofile/controller/performanceprofile/components/utils_test.go index c9c03b8fb7..d4bf9396cc 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/utils_test.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/utils_test.go @@ -33,7 +33,7 @@ var cpuListToHexMask = []listToMask{ } func intersectHelper(cpuListA, cpuListB string) ([]int, error) { - cpuLists, err := NewCPULists(cpuListA, cpuListB, "", "") + cpuLists, err := NewCPULists(cpuListA, cpuListB, "", "", "") if err != nil { return nil, err } diff --git a/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go b/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go index 5e2a26e591..e50ec5229d 100644 --- a/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go +++ b/pkg/performanceprofile/controller/performanceprofile/hypershift/components/handler.go @@ -93,15 +93,18 @@ func (h *handler) Apply(ctx context.Context, obj client.Object, recorder record. klog.Infof("ignoring reconcile loop for pause performance profile %s", profile.Name) return nil } - // set missing options - options.MachineConfig.MixedCPUsEnabled = options.MixedCPUsFeatureGateEnabled && profileutil.IsMixedCPUsEnabled(profile) - options.DRAResourceManagement = profileutil.IsDRAManaged(profile) + + profileutil.SetMissingOptions(profile, options) mfs, err := manifestset.GetNewComponents(profile, options) if err != nil { return err } + if err := profileutil.ValidateDedicatedCPUsPrerequisites(profile, options, mfs.KubeletConfig); err != nil { + return err + } + // get mutated machine config mcMutated, err := resources.GetMutatedMachineConfig(ctx, h.controlPlaneClient, mfs.MachineConfig) if err != nil { diff --git a/pkg/performanceprofile/controller/performanceprofile/status/errors.go b/pkg/performanceprofile/controller/performanceprofile/status/errors.go new file mode 100644 index 0000000000..410f6319dd --- /dev/null +++ b/pkg/performanceprofile/controller/performanceprofile/status/errors.go @@ -0,0 +1,22 @@ +package status + +import "fmt" + +// DedicatedCPUsPrerequisiteError is returned when spec.cpu.dedicated is set +// but neither Workload Partitioning (CPUPartitioningAllNodes) nor the +// strict-cpu-reservation Kubelet CPUManager policy option is enabled. +type DedicatedCPUsPrerequisiteError struct { + Message string +} + +func (e *DedicatedCPUsPrerequisiteError) Error() string { + return e.Message +} + +func NewDedicatedCPUsPrerequisiteError() *DedicatedCPUsPrerequisiteError { + return &DedicatedCPUsPrerequisiteError{ + Message: fmt.Sprintf("dedicated CPUs require either Workload Partitioning (CPUPartitioningAllNodes) " + + "or the strict-cpu-reservation Kubelet CPUManager policy option to be enabled; " + + "without one of these, Burstable and BestEffort QoS pods can still be scheduled on dedicated CPUs"), + } +} diff --git a/pkg/performanceprofile/controller/performanceprofile/status/status.go b/pkg/performanceprofile/controller/performanceprofile/status/status.go index af44969ca1..292876292a 100644 --- a/pkg/performanceprofile/controller/performanceprofile/status/status.go +++ b/pkg/performanceprofile/controller/performanceprofile/status/status.go @@ -32,6 +32,7 @@ const ( ConditionFailedGettingKubeletStatus = "GettingKubeletStatusFailed" ConditionReasonTunedDegraded = "TunedProfileDegraded" ConditionFailedGettingTunedProfileStatus = "GettingTunedStatusFailed" + ConditionDedicatedCPUsPrerequisiteNotMet = "DedicatedCPUsPrerequisiteNotMet" ) type Writer interface { diff --git a/pkg/performanceprofile/controller/performanceprofile_controller.go b/pkg/performanceprofile/controller/performanceprofile_controller.go index 4376c1a18c..78f96f7266 100644 --- a/pkg/performanceprofile/controller/performanceprofile_controller.go +++ b/pkg/performanceprofile/controller/performanceprofile_controller.go @@ -613,6 +613,16 @@ func (r *PerformanceProfileReconciler) Reconcile(ctx context.Context, req ctrl.R klog.V(2).Infof("PerformanceProfile %q: waiting for bootcmdline ready signal, will be triggered by operator controller", instance.GetName()) return reconcile.Result{RequeueAfter: 30 * time.Second}, nil } + if _, isDedicatedPrereq := err.(*status.DedicatedCPUsPrerequisiteError); isDedicatedPrereq { + klog.Errorf("performance profile %q failed dedicated CPUs prerequisite check: %v", instance.GetName(), err) + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "Validation failed", err.Error()) + conditions := status.GetDegradedConditions(status.ConditionDedicatedCPUsPrerequisiteNotMet, err.Error()) + if statusErr := r.StatusWriter.Update(ctx, instance, conditions); statusErr != nil { + klog.Errorf("failed to update performance profile %q status: %v", instance.GetName(), statusErr) + return reconcile.Result{RequeueAfter: statusUpdateRequeueAfter}, nil + } + return reconcile.Result{}, err + } klog.Errorf("failed to deploy performance profile %q components: %v", instance.GetName(), err) r.Recorder.Eventf(instance, corev1.EventTypeWarning, "Creation failed", "Failed to create all components: %v", err) conditions := status.GetDegradedConditions(status.ConditionReasonComponentsCreationFailed, err.Error()) diff --git a/pkg/performanceprofile/controller/performanceprofile_controller_test.go b/pkg/performanceprofile/controller/performanceprofile_controller_test.go index ccc1fad318..b38b3fdab5 100644 --- a/pkg/performanceprofile/controller/performanceprofile_controller_test.go +++ b/pkg/performanceprofile/controller/performanceprofile_controller_test.go @@ -1031,6 +1031,62 @@ var _ = Describe("Controller", func() { }) }) + Context("with dedicated CPUs prerequisite validation", func() { + BeforeEach(skipForHypershift) + + It("should degrade when dedicated CPUs set without workload partitioning or strict-cpu-reservation", func() { + dedicated := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicated + profile.SetFinalizers([]string{openshiftFinalizer}) + + r := newFakeReconciler(profile, profileMCP, infra, clusterOperator) + result, err := r.Reconcile(context.TODO(), request) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("dedicated CPUs require either Workload Partitioning")) + Expect(result).To(Equal(reconcile.Result{})) + + updatedProfile := &performancev2.PerformanceProfile{} + key := types.NamespacedName{ + Name: profile.Name, + Namespace: metav1.NamespaceNone, + } + Expect(r.Get(context.TODO(), key, updatedProfile)).ToNot(HaveOccurred()) + degradedCondition := conditionsv1.FindStatusCondition(updatedProfile.Status.Conditions, conditionsv1.ConditionDegraded) + Expect(degradedCondition).ToNot(BeNil()) + Expect(degradedCondition.Status).To(Equal(corev1.ConditionTrue)) + Expect(degradedCondition.Reason).To(Equal(status.ConditionDedicatedCPUsPrerequisiteNotMet)) + }) + + It("should not degrade when dedicated CPUs set with workload partitioning enabled", func() { + dedicated := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicated + profile.SetFinalizers([]string{openshiftFinalizer}) + infra = testutils.NewInfraResource(true) + + r := newFakeReconciler(profile, profileMCP, infra, clusterOperator) + Expect(reconcileWithBootcmdlineSync(r, request, profileMCP, false)).To(Equal(reconcile.Result{})) + }) + + It("should not degrade when dedicated CPUs set with strict-cpu-reservation annotation", func() { + dedicated := performancev2.CPUSet("10-11") + profile.Spec.CPU.Dedicated = &dedicated + profile.Annotations = map[string]string{ + "kubeletconfig.experimental": `{"cpuManagerPolicyOptions": {"strict-cpu-reservation": "true"}}`, + } + profile.SetFinalizers([]string{openshiftFinalizer}) + + r := newFakeReconciler(profile, profileMCP, infra, clusterOperator) + Expect(reconcileWithBootcmdlineSync(r, request, profileMCP, false)).To(Equal(reconcile.Result{})) + }) + + It("should not degrade when dedicated CPUs are not set", func() { + profile.SetFinalizers([]string{openshiftFinalizer}) + + r := newFakeReconciler(profile, profileMCP, infra, clusterOperator) + Expect(reconcileWithBootcmdlineSync(r, request, profileMCP, false)).To(Equal(reconcile.Result{})) + }) + }) + It("should map machine config pool to the performance profile", func() { skipForHypershift() mcp := &mcov1.MachineConfigPool{ diff --git a/test/e2e/performanceprofile/functests/2_performance_update/dedicatedcpus.go b/test/e2e/performanceprofile/functests/2_performance_update/dedicatedcpus.go new file mode 100644 index 0000000000..404abaa7ae --- /dev/null +++ b/test/e2e/performanceprofile/functests/2_performance_update/dedicatedcpus.go @@ -0,0 +1,390 @@ +//go:build !unittests + +package __performance_update + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/cpuset" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" + testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" + testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cluster" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label" + testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate" +) + +var _ = Describe("[performance] Dedicated CPUs for DPDK", Ordered, Label(string(label.DedicatedCPUs), string(label.Slow), string(label.Tier2)), func() { + var ( + workerRTNodes []corev1.Node + profile *performancev2.PerformanceProfile + initialProfile *performancev2.PerformanceProfile + + reservedSet cpuset.CPUSet + dedicatedSet cpuset.CPUSet + newIsolatedSet cpuset.CPUSet + ) + + BeforeAll(func() { + if discovery.Enabled() && testutils.ProfileNotFound { + Skip("Discovery mode enabled, performance profile not found") + } + + var err error + workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) + Expect(err).ToNot(HaveOccurred()) + Expect(workerRTNodes).ToNot(BeEmpty()) + + profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + initialProfile = profile.DeepCopy() + + reservedSet, err = cpuset.Parse(string(*profile.Spec.CPU.Reserved)) + Expect(err).ToNot(HaveOccurred()) + isolatedSet, err := cpuset.Parse(string(*profile.Spec.CPU.Isolated)) + Expect(err).ToNot(HaveOccurred()) + + isolatedList := isolatedSet.List() + Expect(len(isolatedList)).To(BeNumerically(">=", 2), + "need at least 2 isolated CPUs to split into isolated+dedicated") + + dedicatedSet = cpuset.New(isolatedList[0]) + newIsolatedSet = cpuset.New(isolatedList[1:]...) + + dedicatedCPUs := performancev2.CPUSet(dedicatedSet.String()) + newIsolated := performancev2.CPUSet(newIsolatedSet.String()) + + testlog.Infof("Reserved: %s, Isolated: %s, Dedicated: %s", + reservedSet.String(), newIsolatedSet.String(), dedicatedSet.String()) + + ctx := context.TODO() + isWPEnabled, err := cluster.IsWorkloadPartitioningEnabled(ctx, testclient.Client) + Expect(err).ToNot(HaveOccurred()) + + By("Updating the profile with dedicated CPUs and disableOvsDynamicPinning") + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + currentProfile.Spec.CPU.Isolated = &newIsolated + currentProfile.Spec.CPU.Dedicated = &dedicatedCPUs + if currentProfile.Spec.Net == nil { + currentProfile.Spec.Net = &performancev2.Net{} + } + currentProfile.Spec.Net.DisableOvsDynamicPinning = ptr.To(true) + + policyOptions := map[string]string{} + if !isWPEnabled { + testlog.Infof("Workload partitioning not enabled, adding strict-cpu-reservation via experimental annotation") + policyOptions["strict-cpu-reservation"] = "true" + if currentProfile.Annotations == nil { + currentProfile.Annotations = make(map[string]string) + } + optJSON, err := json.Marshal(map[string]interface{}{"cpuManagerPolicyOptions": policyOptions}) + Expect(err).ToNot(HaveOccurred()) + currentProfile.Annotations["kubeletconfig.experimental"] = string(optJSON) + } + + profiles.UpdateWithRetry(currentProfile) + + updatedProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + testlog.Infof("Updated profile: reserved=%s isolated=%s dedicated=%s annotations=%v", + *updatedProfile.Spec.CPU.Reserved, *updatedProfile.Spec.CPU.Isolated, + *updatedProfile.Spec.CPU.Dedicated, updatedProfile.Annotations) + + By("Waiting for the tuning to be applied") + profilesupdate.WaitForTuningUpdating(ctx, currentProfile) + profilesupdate.WaitForTuningUpdated(ctx, currentProfile) + + By("Refreshing the node list after the update") + workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) + Expect(err).ToNot(HaveOccurred()) + Expect(workerRTNodes).ToNot(BeEmpty()) + }) + + AfterAll(func() { + if initialProfile == nil { + return + } + By("Reverting the profile to its initial state") + ctx := context.TODO() + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + + initialProfile.ResourceVersion = currentProfile.ResourceVersion + profiles.UpdateWithRetry(initialProfile) + + profilesupdate.WaitForTuningUpdating(ctx, initialProfile) + profilesupdate.WaitForTuningUpdated(ctx, initialProfile) + }) + + Context("when dedicated CPUs and disableOvsDynamicPinning are set", func() { + It("should apply dedicated CPU node configuration", func() { + ctx := context.TODO() + expectedIsolatedPlusDedicated := newIsolatedSet.Union(dedicatedSet) + expectedReservedSystem := reservedSet.Union(dedicatedSet) + + for i := range workerRTNodes { + node := &workerRTNodes[i] + testlog.Infof("Verifying node %s", node.Name) + + By(fmt.Sprintf("Verifying kernel cmdline on node %s", node.Name)) + cmdline, err := nodes.ExecCommand(ctx, node, []string{"cat", "/proc/cmdline"}) + Expect(err).ToNot(HaveOccurred()) + cmdlineStr := testutils.ToString(cmdline) + + By(fmt.Sprintf("Verifying isolcpus includes dedicated CPUs on node %s", node.Name)) + isolcpusSet := parseCPUSetFromKernelParam(cmdlineStr, "isolcpus") + Expect(isolcpusSet.IsEmpty()).To(BeFalse(), "isolcpus param not found in cmdline") + Expect(expectedIsolatedPlusDedicated.IsSubsetOf(isolcpusSet)).To(BeTrue(), + fmt.Sprintf("isolcpus=%s should include all isolated+dedicated CPUs %s", + isolcpusSet.String(), expectedIsolatedPlusDedicated.String())) + + By(fmt.Sprintf("Verifying nohz_full includes dedicated CPUs on node %s", node.Name)) + nohzSet := parseCPUSetFromKernelParam(cmdlineStr, "nohz_full") + Expect(nohzSet.IsEmpty()).To(BeFalse(), "nohz_full param not found in cmdline") + Expect(dedicatedSet.IsSubsetOf(nohzSet)).To(BeTrue(), + fmt.Sprintf("nohz_full=%s should include dedicated CPUs %s", + nohzSet.String(), dedicatedSet.String())) + + By(fmt.Sprintf("Verifying rcu_nocbs includes dedicated CPUs on node %s", node.Name)) + rcuSet := parseCPUSetFromKernelParam(cmdlineStr, "rcu_nocbs") + Expect(rcuSet.IsEmpty()).To(BeFalse(), "rcu_nocbs param not found in cmdline") + Expect(dedicatedSet.IsSubsetOf(rcuSet)).To(BeTrue(), + fmt.Sprintf("rcu_nocbs=%s should include dedicated CPUs %s", + rcuSet.String(), dedicatedSet.String())) + + By(fmt.Sprintf("Verifying systemd.cpu_affinity excludes dedicated CPUs on node %s", node.Name)) + affinitySet := parseCPUSetFromKernelParam(cmdlineStr, "systemd.cpu_affinity") + Expect(affinitySet.IsEmpty()).To(BeFalse(), "systemd.cpu_affinity param not found in cmdline") + Expect(affinitySet.Intersection(dedicatedSet).IsEmpty()).To(BeTrue(), + fmt.Sprintf("systemd.cpu_affinity=%s should not contain dedicated CPUs %s", + affinitySet.String(), dedicatedSet.String())) + + By(fmt.Sprintf("Verifying kubelet reservedSystemCPUs is union of reserved+dedicated on node %s", node.Name)) + kubeletConfig, err := nodes.GetKubeletConfig(ctx, node) + Expect(err).ToNot(HaveOccurred()) + reservedSystemCPUs, err := cpuset.Parse(kubeletConfig.ReservedSystemCPUs) + Expect(err).ToNot(HaveOccurred()) + Expect(reservedSystemCPUs.Equals(expectedReservedSystem)).To(BeTrue(), + fmt.Sprintf("ReservedSystemCPUs should be %s (reserved+dedicated), got %s", + expectedReservedSystem.String(), reservedSystemCPUs.String())) + + By(fmt.Sprintf("Verifying IRQBALANCE_BANNED_CPUS on node %s", node.Name)) + irqConf, err := nodes.ExecCommand(ctx, node, []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}) + Expect(err).ToNot(HaveOccurred()) + irqConfStr := testutils.ToString(irqConf) + + bannedSet := parseIRQBannedCPUSet(irqConfStr) + Expect(dedicatedSet.IsSubsetOf(bannedSet)).To(BeTrue(), + fmt.Sprintf("IRQBALANCE_BANNED_CPUS should include dedicated CPUs %s, got %s", + dedicatedSet.String(), bannedSet.String())) + + By(fmt.Sprintf("Verifying default_smp_affinity excludes dedicated CPUs on node %s", node.Name)) + smpAffinity, err := nodes.ExecCommand(ctx, node, []string{"cat", "/proc/irq/default_smp_affinity"}) + Expect(err).ToNot(HaveOccurred()) + smpAffinityStr := strings.TrimSpace(testutils.ToString(smpAffinity)) + testlog.Infof("default_smp_affinity on %s: %s", node.Name, smpAffinityStr) + smpCPUSet := parseHexMaskToCPUSet(smpAffinityStr) + Expect(smpCPUSet.Intersection(dedicatedSet).IsEmpty()).To(BeTrue(), + fmt.Sprintf("default_smp_affinity should not have dedicated CPU bits set, got CPUs %s", + smpCPUSet.Intersection(dedicatedSet).String())) + + By(fmt.Sprintf("Verifying OVS dynamic pinning trigger file is absent on node %s", node.Name)) + _, err = nodes.ExecCommand(ctx, node, []string{ + "ls", "/rootfs/var/lib/ovn-ic/etc/enable_dynamic_cpu_affinity", + }) + Expect(err).To(HaveOccurred(), + fmt.Sprintf("OVS dynamic pinning trigger file should not exist on node %s when disableOvsDynamicPinning=true", node.Name)) + + By(fmt.Sprintf("Verifying dedicatedcpus.slice exists on node %s", node.Name)) + _, err = nodes.ExecCommand(ctx, node, []string{ + "cat", "/rootfs/etc/systemd/system/dedicatedcpus.slice", + }) + Expect(err).ToNot(HaveOccurred(), + "dedicatedcpus.slice should be present on the node") + + By(fmt.Sprintf("Verifying dedicated-cpus-configure script exists on node %s", node.Name)) + _, err = nodes.ExecCommand(ctx, node, []string{ + "cat", "/rootfs/usr/local/bin/dedicated-cpus-configure.sh", + }) + Expect(err).ToNot(HaveOccurred(), + "dedicated-cpus-configure.sh should be present on the node") + + By(fmt.Sprintf("Verifying dedicatedcpus.slice cpuset partition is isolated on node %s", node.Name)) + cgroupPartition, err := nodes.ExecCommand(ctx, node, []string{ + "cat", "/rootfs/sys/fs/cgroup/dedicatedcpus.slice/cpuset.cpus.partition", + }) + Expect(err).ToNot(HaveOccurred(), + "failed to read dedicatedcpus.slice cpuset.cpus.partition") + partitionStr := strings.TrimSpace(testutils.ToString(cgroupPartition)) + Expect(partitionStr).To(Equal("isolated"), + "dedicatedcpus.slice cpuset.cpus.partition should be 'isolated'") + } + }) + + It("should preserve dedicated CPU IRQ banning across GU pod lifecycle", func() { + ctx := context.TODO() + node := &workerRTNodes[0] + testlog.Infof("Testing CRI-O IRQ interaction on node %s", node.Name) + + By("Verifying default_smp_affinity has dedicated CPU bits cleared before pod creation") + smpAffinity, err := nodes.ExecCommand(ctx, node, []string{"cat", "/proc/irq/default_smp_affinity"}) + Expect(err).ToNot(HaveOccurred()) + smpBeforePod := strings.TrimSpace(testutils.ToString(smpAffinity)) + testlog.Infof("default_smp_affinity before pod: %s", smpBeforePod) + smpBeforeSet := parseHexMaskToCPUSet(smpBeforePod) + Expect(smpBeforeSet.Intersection(dedicatedSet).IsEmpty()).To(BeTrue(), + fmt.Sprintf("default_smp_affinity should not have dedicated CPU bits set before pod, got CPUs %s", + smpBeforeSet.Intersection(dedicatedSet).String())) + + By("Verifying IRQBALANCE_BANNED_CPUS is set to dedicated hex mask before pod creation") + irqConf, err := nodes.ExecCommand(ctx, node, []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}) + Expect(err).ToNot(HaveOccurred()) + bannedBeforePod := parseIRQBannedCPUSet(testutils.ToString(irqConf)) + Expect(dedicatedSet.IsSubsetOf(bannedBeforePod)).To(BeTrue(), + fmt.Sprintf("IRQBALANCE_BANNED_CPUS should include dedicated CPUs %s before pod, got %s", + dedicatedSet.String(), bannedBeforePod.String())) + + By("Creating a Guaranteed pod with irq-load-balancing=disable") + testpod := pods.GetTestPod() + testpod.Namespace = testutils.NamespaceTesting + testpod.Annotations = map[string]string{ + "irq-load-balancing.crio.io": "disable", + } + testpod.Spec.Containers[0].Resources = corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + runtimeClassName := components.GetComponentName(profile.Name, components.ComponentNamePrefix) + testpod.Spec.RuntimeClassName = &runtimeClassName + testpod.Spec.NodeSelector = map[string]string{testutils.LabelHostname: node.Name} + + err = testclient.DataPlaneClient.Create(ctx, testpod) + Expect(err).ToNot(HaveOccurred()) + + podKey := client.ObjectKeyFromObject(testpod) + testpod, err = pods.WaitForCondition(ctx, podKey, corev1.PodReady, corev1.ConditionTrue, 10*time.Minute) + pods.DumpStateOnFailure(ctx, testclient.K8sClient, testpod, err) + Expect(err).ToNot(HaveOccurred()) + Expect(testpod.Status.QOSClass).To(Equal(corev1.PodQOSGuaranteed)) + testlog.Infof("GU pod %s is running on node %s", testpod.Name, node.Name) + + By("Verifying IRQBALANCE_BANNED_CPUS still includes dedicated CPUs with pod running") + irqConf, err = nodes.ExecCommand(ctx, node, []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}) + Expect(err).ToNot(HaveOccurred()) + bannedWithPod := parseIRQBannedCPUSet(testutils.ToString(irqConf)) + Expect(dedicatedSet.IsSubsetOf(bannedWithPod)).To(BeTrue(), + fmt.Sprintf("IRQBALANCE_BANNED_CPUS should include dedicated CPUs %s with pod running, got %s", + dedicatedSet.String(), bannedWithPod.String())) + + By("Deleting the GU pod") + err = testclient.DataPlaneClient.Delete(ctx, testpod) + Expect(err).ToNot(HaveOccurred()) + err = pods.WaitForDeletion(ctx, testclient.DataPlaneClient, testpod, 5*time.Minute) + Expect(err).ToNot(HaveOccurred()) + + By("Verifying IRQBALANCE_BANNED_CPUS still includes dedicated CPUs after pod deletion") + Eventually(func() bool { + irqConf, err := nodes.ExecCommand(ctx, node, []string{"cat", "/rootfs/etc/sysconfig/irqbalance"}) + if err != nil { + return false + } + bannedAfterPod := parseIRQBannedCPUSet(testutils.ToString(irqConf)) + return dedicatedSet.IsSubsetOf(bannedAfterPod) + }, 2*time.Minute, 10*time.Second).Should(BeTrue(), + "IRQBALANCE_BANNED_CPUS should still include dedicated CPUs after pod deletion") + + By("Verifying default_smp_affinity still has dedicated CPU bits cleared after pod deletion") + smpAffinity, err = nodes.ExecCommand(ctx, node, []string{"cat", "/proc/irq/default_smp_affinity"}) + Expect(err).ToNot(HaveOccurred()) + smpAfterPod := strings.TrimSpace(testutils.ToString(smpAffinity)) + testlog.Infof("default_smp_affinity after pod deletion: %s", smpAfterPod) + smpAfterSet := parseHexMaskToCPUSet(smpAfterPod) + Expect(smpAfterSet.Intersection(dedicatedSet).IsEmpty()).To(BeTrue(), + fmt.Sprintf("default_smp_affinity should keep dedicated CPU bits cleared after pod deletion, got CPUs %s", + smpAfterSet.Intersection(dedicatedSet).String())) + }) + }) +}) + +// parseCPUSetFromKernelParam extracts the CPU list from a kernel cmdline +// parameter like "isolcpus=managed_irq,1-5" or "nohz_full=2-5" and returns it +// as a cpuset.CPUSet. For isolcpus, non-numeric flag prefixes (e.g. +// "managed_irq,") are stripped before parsing. +func parseCPUSetFromKernelParam(cmdline, param string) cpuset.CPUSet { + for _, field := range strings.Fields(cmdline) { + if !strings.HasPrefix(field, param+"=") { + continue + } + val := strings.TrimPrefix(field, param+"=") + for i, c := range val { + if c >= '0' && c <= '9' { + val = val[i:] + break + } + } + set, err := cpuset.Parse(val) + if err != nil { + return cpuset.New() + } + return set + } + return cpuset.New() +} + +// parseHexMaskToCPUSet converts a comma-separated hex bitmask (e.g. +// "ff,fffffffe" or "00000040") into a cpuset.CPUSet by setting each +// CPU whose bit is 1 in the mask. +func parseHexMaskToCPUSet(mask string) cpuset.CPUSet { + cleaned := strings.ReplaceAll(mask, ",", "") + n := new(big.Int) + if _, ok := n.SetString(cleaned, 16); !ok { + return cpuset.New() + } + var cpus []int + for i := 0; i < n.BitLen(); i++ { + if n.Bit(i) == 1 { + cpus = append(cpus, i) + } + } + return cpuset.New(cpus...) +} + +// parseIRQBannedCPUSet extracts the IRQBALANCE_BANNED_CPUS value from the +// irqbalance config content and returns it as a cpuset.CPUSet. +func parseIRQBannedCPUSet(irqbalanceContent string) cpuset.CPUSet { + for _, line := range strings.Split(irqbalanceContent, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "IRQBALANCE_BANNED_CPUS=") { + continue + } + val := strings.TrimPrefix(line, "IRQBALANCE_BANNED_CPUS=") + val = strings.Trim(val, `"`) + return parseHexMaskToCPUSet(val) + } + return cpuset.New() +} diff --git a/test/e2e/performanceprofile/functests/3_performance_status/status.go b/test/e2e/performanceprofile/functests/3_performance_status/status.go index 0453de0d48..9a56ee38e7 100644 --- a/test/e2e/performanceprofile/functests/3_performance_status/status.go +++ b/test/e2e/performanceprofile/functests/3_performance_status/status.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/utils/cpuset" "sigs.k8s.io/controller-runtime/pkg/client" ign2types "github.com/coreos/ignition/config/v2_2/types" @@ -29,6 +30,7 @@ import ( hypershiftconsts "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/hypershift/consts" testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/cluster" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/discovery" hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label" @@ -212,6 +214,59 @@ var _ = Describe("Status testing of performance profile", Ordered, func() { Expect(nodepools.WaitForConfigToBeReady(ctx, testclient.ControlPlaneClient, np.Name, np.Namespace)).To(Succeed()) }) }) + + Context("Dedicated CPUs prerequisites", Label(string(label.DedicatedCPUs), string(label.OpenShift), string(label.Tier2)), func() { + It("should report Degraded when workload partitioning is disabled", func() { + ctx := context.TODO() + + isWPEnabled, err := cluster.IsWorkloadPartitioningEnabled(ctx, testclient.Client) + Expect(err).ToNot(HaveOccurred()) + if isWPEnabled { + Skip("Workload partitioning is enabled, skipping Degraded prerequisite test") + } + + profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + initialSpec := *profile.Spec.DeepCopy() + + isolatedSet, err := cpuset.Parse(string(*profile.Spec.CPU.Isolated)) + Expect(err).ToNot(HaveOccurred()) + isolatedList := isolatedSet.List() + Expect(len(isolatedList)).To(BeNumerically(">=", 2), + "need at least 2 isolated CPUs to designate as dedicated") + + dedicatedSet := cpuset.New(isolatedList[0]) + remainingIsolated := cpuset.New(isolatedList[1:]...) + dedicatedCPUs := performancev2.CPUSet(dedicatedSet.String()) + newIsolated := performancev2.CPUSet(remainingIsolated.String()) + + By("Setting dedicated CPUs on the profile without workload partitioning") + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + currentProfile.Spec.CPU.Isolated = &newIsolated + currentProfile.Spec.CPU.Dedicated = &dedicatedCPUs + profiles.UpdateWithRetry(currentProfile) + + defer func() { + By("Reverting the profile to its initial state") + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + currentProfile.Spec = initialSpec + currentProfile.Spec.CPU.Dedicated = nil + profiles.UpdateWithRetry(currentProfile) + profiles.WaitForCondition(testutils.NodeSelectorLabels, v1.ConditionAvailable, corev1.ConditionTrue) + }() + + By("Waiting for Degraded condition") + profiles.WaitForCondition(testutils.NodeSelectorLabels, v1.ConditionDegraded, corev1.ConditionTrue) + + By("Verifying the Degraded message contains the prerequisite error") + conditionMessage := profiles.GetConditionMessage(testutils.NodeSelectorLabels, v1.ConditionDegraded) + Expect(conditionMessage).To(ContainSubstring( + "dedicated CPUs require either Workload Partitioning (CPUPartitioningAllNodes) " + + "or the strict-cpu-reservation Kubelet CPUManager policy option to be enabled")) + }) + }) }) func createBadMachineConfig(name string) *machineconfigv1.MachineConfig { diff --git a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go index 7942c3d8fc..e0ed952a65 100644 --- a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go +++ b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/utils/cpuset" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" @@ -96,7 +97,7 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab ovsSystemdServices = ovsSystemdServicesOnOvsSlice(ctx, workerRTNode) - isWorkloadPartitioningEnabled, err = cluster.IsWorkloadPartitioningEnabled(ctx) + isWorkloadPartitioningEnabled, err = cluster.IsWorkloadPartitioningEnabled(ctx, testclient.Client) Expect(err).ToNot(HaveOccurred(), "Unable to check if workload partitioning is enabled") testlog.Infof("Workload partitioning enabled: %v", isWorkloadPartitioningEnabled) @@ -698,6 +699,7 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab err := testclient.DataPlaneClient.Create(ctx, testpod) Expect(err).ToNot(HaveOccurred()) testpod, err = pods.WaitForCondition(ctx, client.ObjectKeyFromObject(testpod), corev1.PodReady, corev1.ConditionTrue, 5*time.Minute) + pods.DumpStateOnFailure(ctx, testclient.K8sClient, testpod, err) Expect(err).ToNot(HaveOccurred()) Expect(testpod.Status.QOSClass).To(Equal(corev1.PodQOSGuaranteed)) guPods = append(guPods, testpod) @@ -740,6 +742,73 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab }) }) + Context("disableOvsDynamicPinning", Label(string(label.DedicatedCPUs), string(label.Tier2)), func() { + It("should remove the activation file and stop dynamic affinity adjustment", func() { + By("Verifying the activation file exists before disabling") + cmd := []string{"ls", activation_file} + _, err := nodes.ExecCommand(ctx, workerRTNode, cmd) + Expect(err).ToNot(HaveOccurred(), "activation file should exist before disabling OVS dynamic pinning") + + By("Capturing OVS service affinity before disabling") + ovsAffinitiesBefore := getOvsAffinities(ctx, ovsSystemdServices, workerRTNode) + Expect(ovsAffinitiesBefore).ToNot(BeEmpty(), "expected non-empty OVS affinities") + + By("Setting disableOvsDynamicPinning=true on the profile") + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + savedSpec := *currentProfile.Spec.DeepCopy() + + if currentProfile.Spec.Net == nil { + currentProfile.Spec.Net = &performancev2.Net{} + } + currentProfile.Spec.Net.DisableOvsDynamicPinning = ptr.To(true) + profiles.UpdateWithRetry(currentProfile) + + profilesupdate.WaitForTuningUpdating(ctx, currentProfile) + profilesupdate.WaitForTuningUpdated(ctx, currentProfile) + + By("Verifying the activation file is absent after disabling") + _, err = nodes.ExecCommand(ctx, workerRTNode, cmd) + Expect(err).To(HaveOccurred(), + "activation file should not exist when disableOvsDynamicPinning=true") + + By("Verifying OVS processes retain their systemd service affinity") + ovsAffinitiesAfter := getOvsAffinities(ctx, ovsSystemdServices, workerRTNode) + for pid, mask := range ovsAffinitiesAfter { + testlog.Infof("OVS pid %s affinity with dynamic pinning disabled: %s", pid, mask) + } + + By("Creating a GU pod and verifying OVS affinity does not change") + guPod := createGuPod(ctx, workerRTNode) + ovsAffinitiesWithPod := getOvsAffinities(ctx, ovsSystemdServices, workerRTNode) + for pid, mask := range ovsAffinitiesWithPod { + testlog.Infof("OVS pid %s affinity with GU pod (dynamic pinning disabled): %s", pid, mask) + if afterMask, ok := ovsAffinitiesAfter[pid]; ok { + Expect(mask.Equals(afterMask)).To(BeTrue(), + fmt.Sprintf("OVS pid %s affinity should not change with dynamic pinning disabled (before: %s, after: %s)", + pid, afterMask, mask)) + } + } + + By("Deleting the GU pod") + Expect(pods.DeleteAndSync(ctx, testclient.DataPlaneClient, guPod)).To(Succeed()) + + By("Reverting the profile to re-enable OVS dynamic pinning") + currentProfile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + currentProfile.Spec = savedSpec + profiles.UpdateWithRetry(currentProfile) + + profilesupdate.WaitForTuningUpdating(ctx, currentProfile) + profilesupdate.WaitForTuningUpdated(ctx, currentProfile) + + By("Verifying the activation file is back after re-enabling") + _, err = nodes.ExecCommand(ctx, workerRTNode, cmd) + Expect(err).ToNot(HaveOccurred(), + "activation file should exist after re-enabling OVS dynamic pinning") + }) + }) + }) }) @@ -1040,6 +1109,7 @@ func createGuPod(ctx context.Context, node *corev1.Node) *corev1.Pod { err := testclient.DataPlaneClient.Create(ctx, testpod) Expect(err).ToNot(HaveOccurred()) testpod, err = pods.WaitForCondition(ctx, client.ObjectKeyFromObject(testpod), corev1.PodReady, corev1.ConditionTrue, 5*time.Minute) + pods.DumpStateOnFailure(ctx, testclient.K8sClient, testpod, err) Expect(err).ToNot(HaveOccurred()) Expect(testpod.Status.QOSClass).To(Equal(corev1.PodQOSGuaranteed)) testlog.Infof("GU pod %s pinned to cpus %s", testpod.Name, getGuPodCPUs(ctx, testpod)) diff --git a/test/e2e/performanceprofile/functests/utils/cluster/cluster.go b/test/e2e/performanceprofile/functests/utils/cluster/cluster.go index 03ce1a9ea0..6c11da957f 100644 --- a/test/e2e/performanceprofile/functests/utils/cluster/cluster.go +++ b/test/e2e/performanceprofile/functests/utils/cluster/cluster.go @@ -41,9 +41,12 @@ func IsControlPlaneSchedulable(ctx context.Context) (bool, error) { // IsWorkloadPartitioningEnabled checks whether CPU partitioning is enabled // cluster-wide by querying the Infrastructure resource's CPUPartitioning status. -func IsWorkloadPartitioningEnabled(ctx context.Context) (bool, error) { +// The caller must pass the appropriate client: on HyperShift the controller +// reads the hosted cluster's Infrastructure, so tests should use the data-plane +// client (testclient.Client) to match. +func IsWorkloadPartitioningEnabled(ctx context.Context, cli client.Client) (bool, error) { infra := &configv1.Infrastructure{} - if err := testclient.ControlPlaneClient.Get(ctx, client.ObjectKey{Name: "cluster"}, infra); err != nil { + if err := cli.Get(ctx, client.ObjectKey{Name: "cluster"}, infra); err != nil { return false, err } return infra.Status.CPUPartitioning == configv1.CPUPartitioningAllNodes, nil diff --git a/test/e2e/performanceprofile/functests/utils/label/label.go b/test/e2e/performanceprofile/functests/utils/label/label.go index b0a2da732f..c33c1abf1c 100644 --- a/test/e2e/performanceprofile/functests/utils/label/label.go +++ b/test/e2e/performanceprofile/functests/utils/label/label.go @@ -70,6 +70,9 @@ const ( // ControlplaneSched features be added in tests that test when control plane schedulable is enabled CtrlPlaneSchedulable Feature = "controlplane-schedulable" + + // DedicatedCPUs should be added in tests that test dedicated CPUs for DPDK vSwitch/vRouter + DedicatedCPUs Feature = "dedicated-cpus" ) // Tier is a label to classify tests under specific grade/level diff --git a/test/e2e/performanceprofile/functests/utils/pods/pods.go b/test/e2e/performanceprofile/functests/utils/pods/pods.go index 2763fb2b62..2a87f394bb 100644 --- a/test/e2e/performanceprofile/functests/utils/pods/pods.go +++ b/test/e2e/performanceprofile/functests/utils/pods/pods.go @@ -28,6 +28,7 @@ import ( testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/events" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/images" + testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles" ) @@ -340,3 +341,43 @@ func CheckPODSchedulingFailed(c client.Client, pod *corev1.Pod) (bool, error) { } return false, nil } + +// DumpPodStateOnFailure logs detailed diagnostics for a pod that failed to +// reach the expected state: phase, conditions, container states, events, and +// logs. It is a no-op when err is nil. +func DumpStateOnFailure(ctx context.Context, k8sClient *kubernetes.Clientset, pod *corev1.Pod, err error) { + if err == nil { + return + } + testlog.Infof("Dumping pod %s/%s state on failure: phase=%s node=%s reason=%s message=%s", + pod.Namespace, pod.Name, pod.Status.Phase, pod.Spec.NodeName, pod.Status.Reason, pod.Status.Message) + + for _, cond := range pod.Status.Conditions { + if cond.Status != corev1.ConditionTrue { + testlog.Infof(" condition %s=%s reason=%s message=%s", cond.Type, cond.Status, cond.Reason, cond.Message) + } + } + + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil { + testlog.Infof(" container %s waiting: reason=%s message=%s", + cs.Name, cs.State.Waiting.Reason, cs.State.Waiting.Message) + } + if cs.State.Terminated != nil { + testlog.Infof(" container %s terminated: reason=%s message=%s exitCode=%d", + cs.Name, cs.State.Terminated.Reason, cs.State.Terminated.Message, cs.State.Terminated.ExitCode) + } + } + + events, err := k8sClient.CoreV1().Events(pod.Namespace).List(ctx, + metav1.ListOptions{FieldSelector: fmt.Sprintf("involvedObject.name=%s", pod.Name)}) + if err == nil { + for _, evt := range events.Items { + testlog.Infof(" event: %s %s: %s", evt.Type, evt.Reason, evt.Message) + } + } + + if logs, err := GetLogs(k8sClient, pod); err == nil && logs != "" { + testlog.Infof("Pod logs:\n%s", logs) + } +} diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml index 4e1609526c..6f34cf57ca 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml index 50f14cedc5..714811ee44 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml index 4e1609526c..6f34cf57ca 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml index 50f14cedc5..714811ee44 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml index cb39398c2a..3bbbb68b25 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/arm/manual_machineconfig.yaml @@ -34,7 +34,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml index 07790c9db0..70e3f4cd6b 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml @@ -34,7 +34,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml index ed4a6b0137..8e66831a46 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml index ed4a6b0137..8e66831a46 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/pp-norps/manual_machineconfig.yaml @@ -35,7 +35,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448 diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml index e37f7ef1de..e4de7c9be2 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml @@ -34,7 +34,7 @@ spec: path: /usr/local/bin/set-cpus-offline.sh user: {} - contents: - source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7Tk9ORX0iID4+ICIke0lSUUJBTEFOQ0VfQ09ORn0iCgojIHdlIG5vdyBvd24gdGhpcyBjb25maWd1cmF0aW9uLiBCdXQgQ1JJLU8gaGFzIGNvZGUgdG8gcmVzdG9yZSB0aGUgY29uZmlndXJhdGlvbiwKIyBhbmQgdW50aWwgaXQgZ2FpbnMgdGhlIG9wdGlvbiB0byBkaXNhYmxlIHRoaXMgcmVzdG9yZSBmbG93LCB3ZSBuZWVkIHRvIG1ha2UKIyB0aGUgY29uZmlndXJhdGlvbiBjb25zaXN0ZW50IHN1Y2ggYXMgdGhlIENSSS1PIHJlc3RvcmUgd2lsbCBkbyBub3RoaW5nLgppZiBbIC1uICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iIF0gJiYgWyAtZiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdOyB0aGVuCgllY2hvICIke05PTkV9IiA+ICIke0NSSU9fT1JJR19CQU5ORURfQ1BVU30iCmZpCg== + source: data:text/plain;charset=utf-8;base64,IyEvdXNyL2Jpbi9lbnYgYmFzaApzZXQgLWV1byBwaXBlZmFpbApzZXQgLXgKCiMgY29uc3QKU0VEPSIvdXNyL2Jpbi9zZWQiCiMgdHVuYWJsZSAtIG92ZXJyaWRhYmxlIGZvciB0ZXN0aW5nIHB1cnBvc2VzCklSUUJBTEFOQ0VfQ09ORj0iJHsxOi0vZXRjL3N5c2NvbmZpZy9pcnFiYWxhbmNlfSIKQ1JJT19PUklHX0JBTk5FRF9DUFVTPSIkezI6LS9ldGMvc3lzY29uZmlnL29yaWdfaXJxX2Jhbm5lZF9jcHVzfSIKTk9ORT0wCgojIEJBTk5FRF9DUFVTOiB0aGUgZmluYWwgaGV4IG1hc2sgd3JpdHRlbiB0byBJUlFCQUxBTkNFX0JBTk5FRF9DUFVTLgojICAgMS4gSWYgREVESUNBVEVEX0NQVVMgaXMgc2V0ICh2aWEgc3lzdGVtZCBFbnZpcm9ubWVudCksIHVzZSBpdC4KIyAgIDIuIE90aGVyd2lzZSwgZGVmYXVsdCB0byAwIChubyBDUFVzIGJhbm5lZCwgYWxsIHBhcnRpY2lwYXRlIGluIGJhbGFuY2luZykuCmlmIFsgLW4gIiR7REVESUNBVEVEX0NQVVM6LX0iIF07IHRoZW4KCUJBTk5FRF9DUFVTPSIke0RFRElDQVRFRF9DUFVTfSIKZWxzZQoJQkFOTkVEX0NQVVM9IiR7Tk9ORX0iCmZpCgpbICEgLWYgIiR7SVJRQkFMQU5DRV9DT05GfSIgXSAmJiBleGl0IDAKCiR7U0VEfSAtaSAnL15ccypJUlFCQUxBTkNFX0JBTk5FRF9DUFVTXGIvZCcgIiR7SVJRQkFMQU5DRV9DT05GfSIgfHwgZXhpdCAwCiMgQ1BVIG51bWJlcnMgd2hpY2ggaGF2ZSB0aGVpciBjb3JyZXNwb25kaW5nIGJpdHMgc2V0IHRvIG9uZSBpbiB0aGlzIG1hc2sKIyB3aWxsIG5vdCBoYXZlIGFueSBpcnEncyBhc3NpZ25lZCB0byB0aGVtIG9uIHJlYmFsYW5jZS4KIyBzbyB6ZXJvIG1lYW5zIGFsbCBjcHVzIGFyZSBwYXJ0aWNpcGF0aW5nIGluIGxvYWQgYmFsYW5jaW5nLgplY2hvICJJUlFCQUxBTkNFX0JBTk5FRF9DUFVTPSR7QkFOTkVEX0NQVVN9IiA+PiAiJHtJUlFCQUxBTkNFX0NPTkZ9IgoKIyB3ZSBub3cgb3duIHRoaXMgY29uZmlndXJhdGlvbi4gQnV0IENSSS1PIGhhcyBjb2RlIHRvIHJlc3RvcmUgdGhlIGNvbmZpZ3VyYXRpb24sCiMgYW5kIHVudGlsIGl0IGdhaW5zIHRoZSBvcHRpb24gdG8gZGlzYWJsZSB0aGlzIHJlc3RvcmUgZmxvdywgd2UgbmVlZCB0byBtYWtlCiMgdGhlIGNvbmZpZ3VyYXRpb24gY29uc2lzdGVudCBzdWNoIGFzIHRoZSBDUkktTyByZXN0b3JlIHdpbGwgZG8gbm90aGluZy4KaWYgWyAtbiAiJHtDUklPX09SSUdfQkFOTkVEX0NQVVN9IiBdICYmIFsgLWYgIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIgXTsgdGhlbgoJZWNobyAiJHtCQU5ORURfQ1BVU30iID4gIiR7Q1JJT19PUklHX0JBTk5FRF9DUFVTfSIKZmkKCiMgQ1JJLU8gcmVhZHMgL3Byb2MvaXJxL2RlZmF1bHRfc21wX2FmZmluaXR5IHRvIGRlcml2ZSB0aGUgSVJRIGJhbm5lZCBtYXNrCiMgd2hlbiBwb2RzIHdpdGggaXJxLWxvYWQtYmFsYW5jaW5nLmNyaW8uaW89ZGlzYWJsZSBhcmUgc2NoZWR1bGVkLgojIElmIHdlIGRvbid0IHJlbW92ZSB0aGUgZGVkaWNhdGVkIENQVXMgZnJvbSBkZWZhdWx0X3NtcF9hZmZpbml0eSBoZXJlLAojIENSSS1PIHdpbGwgb3ZlcndyaXRlIElSUUJBTEFOQ0VfQkFOTkVEX0NQVVMgd2hpbGUgaWdub3JpbmcgdGhlIGRlZGljYXRlZCBDUFVzLgpTTVBfQUZGSU5JVFk9Ii9wcm9jL2lycS9kZWZhdWx0X3NtcF9hZmZpbml0eSIKaWYgWyAiJHtCQU5ORURfQ1BVU30iICE9ICIke05PTkV9IiBdICYmIFsgLWYgIiR7U01QX0FGRklOSVRZfSIgXTsgdGhlbgoJIyBkZWZhdWx0X3NtcF9hZmZpbml0eSBpcyBjb21tYS1zZXBhcmF0ZWQgMzItYml0IGhleCBncm91cHMgKGUuZy4gImZmLGZmZmZmZmZmLGZmZmZmZmZmIikKCUlGUz0nLCcgcmVhZCAtcmEgc21wIDwgIiR7U01QX0FGRklOSVRZfSIKCW49JHsjc21wW0BdfQoJIyBwYWQgQkFOTkVEX0NQVVMgd2l0aCBsZWFkaW5nIHplcm9zIHRvIG1hdGNoIHRoZSBzYW1lIG51bWJlciBvZiBoZXggY2hhcnMKCXBhZGRlZD0iJHtCQU5ORURfQ1BVU30iCgl3aGlsZSBbICR7I3BhZGRlZH0gLWx0ICQoKCBuICogOCApKSBdOyBkbwoJCXBhZGRlZD0iMCR7cGFkZGVkfSIKCWRvbmUKCSMgY2xlYXIgYmFubmVkIGJpdHMgZnJvbSBlYWNoIDMyLWJpdCBncm91cDogcmVzdWx0ID0gc21wICYgfmJhbm5lZAoJcmVzdWx0PSIiCglmb3IgKCggaT0wOyBpPG47IGkrKyApKTsgZG8KCQliYW49IiR7cGFkZGVkOiQoKCBpICogOCApKTo4fSIKCQl2YWw9JChwcmludGYgIiUwOHgiICQoKCAweCR7c21wWyRpXX0gJiB+MHgke2Jhbn0gKSkpCgkJcmVzdWx0Kz0iJHtyZXN1bHQ6Kyx9JHt2YWx9IgoJZG9uZQoJZWNobyAiU2V0dGluZyBkZWZhdWx0X3NtcF9hZmZpbml0eSB0byAke3Jlc3VsdH0gKHJlbW92aW5nIGRlZGljYXRlZCBDUFVzICR7QkFOTkVEX0NQVVN9IGZyb20gZGVmYXVsdF9zbXBfYWZmaW5pdHkgbWFzaykiCgllY2hvICIke3Jlc3VsdH0iID4gIiR7U01QX0FGRklOSVRZfSIKZmkK verification: {} group: {} mode: 448