From 803ebcb03b28baa24e5bd6a93c4d3d51c10c9cdb Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 15:27:16 +0000 Subject: [PATCH 01/21] chore: add agentbaker artifact streaming combo e2es --- e2e/config/vhd.go | 21 +++++ e2e/scenario_test.go | 219 +++++++++++++++++++++++++++++++++++++++++++ e2e/test_helpers.go | 36 +++++++ 3 files changed, 276 insertions(+) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 43d71f4c281..21fe3472d76 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -60,6 +60,13 @@ var ( Distro: datamodel.AKSUbuntuContainerd2204Gen2, Gallery: imageGalleryLinux, } + VHDUbuntu2204Gen2TLContainerd = &Image{ + Name: "2204gen2TLcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2204TLGen2, + Gallery: imageGalleryLinux, + } VHDUbuntu2204FIPSContainerd = &Image{ Name: "2204fipscontainerd", OS: OSUbuntu, @@ -98,6 +105,20 @@ var ( Distro: datamodel.AKSAzureLinuxV3Gen2, Gallery: imageGalleryLinux, } + VHDAzureLinuxV3CVMGen2 = &Image{ + Name: "AzureLinuxV3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: imageGalleryLinux, + } + VHDAzureLinuxV3KataGen2 = &Image{ + Name: "AzureLinuxV3katagen2", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3Gen2Kata, + Gallery: imageGalleryLinux, + } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 6d0648e2a8d..fa85baedc42 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1314,6 +1314,225 @@ func Test_Ubuntu2404_ArtifactStreaming_ARM64_Scriptless(t *testing.T) { }) } +func Test_Ubuntu2204_ArtifactStreaming_TrustedLaunch(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 node using artifact streaming with trusted launch can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2TLContainerd, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_Ubuntu2204_ArtifactStreaming_TrustedLaunch_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 node using artifact streaming with trusted launch can be properly bootstrapped", + Tags: Tags{ + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2TLContainerd, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties = addTrustedLaunchToVMSS(vmss.Properties) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_Ubuntu2204_ArtifactStreaming_FIPS(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 FIPS node using artifact streaming can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2FIPSContainerd, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ + EnableFips1403Encryption: to.Ptr(true), + } + settings := vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.ProtectedSettings + vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Settings = settings + vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.ProtectedSettings = nil + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_Ubuntu2204_ArtifactStreaming_FIPS_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 FIPS node using artifact streaming can be properly bootstrapped", + Tags: Tags{ + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDUbuntu2204Gen2FIPSContainerd, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.Properties.AdditionalCapabilities = &armcompute.AdditionalCapabilities{ + EnableFips1403Encryption: to.Ptr(true), + } + settings := vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.ProtectedSettings + vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.Settings = settings + vmss.Properties.VirtualMachineProfile.ExtensionProfile.Extensions[0].Properties.ProtectedSettings = nil + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_AzureLinuxV3_ArtifactStreaming_CVM(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 CVM node using artifact streaming can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3CVMGen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + nbc.AgentPoolProfile.VMSize = "Standard_DC2as_v5" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_DC2as_v5") + vmss.Properties = addConfidentialVMToVMSS(vmss.Properties) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_AzureLinuxV3_ArtifactStreaming_CVM_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 CVM node using artifact streaming can be properly bootstrapped", + Tags: Tags{ + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3CVMGen2, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + config.VmSize = "Standard_DC2as_v5" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_DC2as_v5") + vmss.Properties = addConfidentialVMToVMSS(vmss.Properties) + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3KataGen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + nbc.AgentPoolProfile.VMSize = "Standard_D4s_v3" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") + ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") + ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") + }, + }, + }) +} + +func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", + Tags: Tags{ + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3KataGen2, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + config.IsKata = true + config.VmSize = "Standard_D4s_v3" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") + ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") + ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") + }, + }, + }) +} + func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations", diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 71312b8b108..753e27fb558 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -548,6 +548,42 @@ func addTrustedLaunchToVMSS(properties *armcompute.VirtualMachineScaleSetPropert return properties } +func addConfidentialVMToVMSS(properties *armcompute.VirtualMachineScaleSetProperties) *armcompute.VirtualMachineScaleSetProperties { + if properties == nil { + properties = &armcompute.VirtualMachineScaleSetProperties{} + } + + if properties.VirtualMachineProfile == nil { + properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} + } + + if properties.VirtualMachineProfile.SecurityProfile == nil { + properties.VirtualMachineProfile.SecurityProfile = &armcompute.SecurityProfile{} + } + + properties.VirtualMachineProfile.SecurityProfile.SecurityType = to.Ptr(armcompute.SecurityTypesConfidentialVM) + if properties.VirtualMachineProfile.SecurityProfile.UefiSettings == nil { + properties.VirtualMachineProfile.SecurityProfile.UefiSettings = &armcompute.UefiSettings{} + } + properties.VirtualMachineProfile.SecurityProfile.UefiSettings.SecureBootEnabled = to.Ptr(true) + properties.VirtualMachineProfile.SecurityProfile.UefiSettings.VTpmEnabled = to.Ptr(true) + + if properties.VirtualMachineProfile.StorageProfile == nil { + properties.VirtualMachineProfile.StorageProfile = &armcompute.VirtualMachineScaleSetStorageProfile{} + } + if properties.VirtualMachineProfile.StorageProfile.OSDisk == nil { + properties.VirtualMachineProfile.StorageProfile.OSDisk = &armcompute.VirtualMachineScaleSetOSDisk{} + } + if properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk == nil { + properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} + } + properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile = &armcompute.VMDiskSecurityProfile{ + SecurityEncryptionType: to.Ptr(armcompute.SecurityEncryptionTypesVMGuestStateOnly), + } + + return properties +} + func createVMExtensionLinuxAKSNode(ctx context.Context, location *string) (*armcompute.VirtualMachineScaleSetExtension, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() From 6435b97e1f518f7c5df8c45d763d9237f2fa929d Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 16:57:54 +0000 Subject: [PATCH 02/21] copy encryption in e2e image replication --- e2e/config/azure.go | 13 +++++++++++++ e2e/scenario_test.go | 8 ++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index d0de6f04619..fd66a96d42a 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -642,10 +642,23 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, if err != nil { return fmt.Errorf("create a new images client: %v", err) } + + // Copy encryption settings from an existing target region if present. + // This is required for CVM images where Azure mandates the regional + // encryption.osDiskImage.securityProfile.type on every target region. + var encryption *armcompute.EncryptionImages + for _, existing := range version.Properties.PublishingProfile.TargetRegions { + if existing.Encryption != nil { + encryption = existing.Encryption + break + } + } + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), + Encryption: encryption, }) resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index fa85baedc42..886830cabe9 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1483,10 +1483,10 @@ func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing(t *testing.T) { VHD: config.VHDAzureLinuxV3KataGen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.EnableArtifactStreaming = true - nbc.AgentPoolProfile.VMSize = "Standard_D4s_v3" + nbc.AgentPoolProfile.VMSize = "Standard_D4ds_v5" }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") }, Validator: func(ctx context.Context, s *Scenario) { ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") @@ -1514,10 +1514,10 @@ func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing_Scriptless(t *testing.T) AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { config.EnableArtifactStreaming = true config.IsKata = true - config.VmSize = "Standard_D4s_v3" + config.VmSize = "Standard_D4ds_v5" }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") }, Validator: func(ctx context.Context, s *Scenario) { ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") From 33a0aa50301b9e0666d9755031c099e55826f4d6 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 17:51:25 +0000 Subject: [PATCH 03/21] nonpersistedtpm for cvm --- e2e/config/azure.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index fd66a96d42a..ad43db20def 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -654,6 +654,28 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, } } + // For ConfidentialVM images, Azure only allows cross-region replication when + // the encryption type is NonPersistedTPM. Detect CVM images by checking if + // any existing target region has a CVM security profile, and override the + // encryption for the new region to use NonPersistedTPM. + isCVM := encryption != nil && encryption.OSDiskImage != nil && encryption.OSDiskImage.SecurityProfile != nil + if isCVM { + nonPersistedTPMEncryption := &armcompute.EncryptionImages{ + OSDiskImage: &armcompute.OSDiskImageEncryption{ + SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ + ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), + }, + }, + } + encryption = nonPersistedTPMEncryption + // Update all existing target regions to use NonPersistedTPM as well, + // since the PUT replaces the entire target regions list and Azure + // requires consistent CVM encryption for cross-region replication. + for _, existing := range version.Properties.PublishingProfile.TargetRegions { + existing.Encryption = nonPersistedTPMEncryption + } + } + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), From 1b9462b0f431d55a19621e269d5fac276df068ae Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 20:36:25 +0000 Subject: [PATCH 04/21] remove kata --- e2e/config/azure.go | 13 +++------- e2e/config/vhd.go | 7 ------ e2e/scenario_test.go | 58 -------------------------------------------- 3 files changed, 3 insertions(+), 75 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index ad43db20def..581dbd746fa 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -656,24 +656,17 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, // For ConfidentialVM images, Azure only allows cross-region replication when // the encryption type is NonPersistedTPM. Detect CVM images by checking if - // any existing target region has a CVM security profile, and override the - // encryption for the new region to use NonPersistedTPM. + // any existing target region has a CVM security profile, and set NonPersistedTPM + // only on the new region (existing regions' encryption cannot be changed). isCVM := encryption != nil && encryption.OSDiskImage != nil && encryption.OSDiskImage.SecurityProfile != nil if isCVM { - nonPersistedTPMEncryption := &armcompute.EncryptionImages{ + encryption = &armcompute.EncryptionImages{ OSDiskImage: &armcompute.OSDiskImageEncryption{ SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), }, }, } - encryption = nonPersistedTPMEncryption - // Update all existing target regions to use NonPersistedTPM as well, - // since the PUT replaces the entire target regions list and Azure - // requires consistent CVM encryption for cross-region replication. - for _, existing := range version.Properties.PublishingProfile.TargetRegions { - existing.Encryption = nonPersistedTPMEncryption - } } version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 21fe3472d76..d0492de6500 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -112,13 +112,6 @@ var ( Distro: datamodel.AKSAzureLinuxV3CVMGen2, Gallery: imageGalleryLinux, } - VHDAzureLinuxV3KataGen2 = &Image{ - Name: "AzureLinuxV3katagen2", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3Gen2Kata, - Gallery: imageGalleryLinux, - } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 886830cabe9..0ac1bd0dee0 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1475,64 +1475,6 @@ func Test_AzureLinuxV3_ArtifactStreaming_CVM_Scriptless(t *testing.T) { }) } -func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3KataGen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.EnableArtifactStreaming = true - nbc.AgentPoolProfile.VMSize = "Standard_D4ds_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") - ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") - ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") - }, - }, - }) -} - -func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", - Tags: Tags{ - Scriptless: true, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3KataGen2, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.EnableArtifactStreaming = true - config.IsKata = true - config.VmSize = "Standard_D4ds_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") - ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") - ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") - }, - }, - }) -} - func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations", From 0c80ae64bb8de990e59e303b711cf43cbd2da734 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 21:30:41 +0000 Subject: [PATCH 05/21] fix cvm --- e2e/config/azure.go | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 581dbd746fa..32fbd87f08a 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -644,8 +644,10 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, } // Copy encryption settings from an existing target region if present. - // This is required for CVM images where Azure mandates the regional - // encryption.osDiskImage.securityProfile.type on every target region. + // This is required for CVM images where Azure mandates that all target + // regions have matching encryption configuration. We copy the existing + // encryption verbatim (including DiskEncryptionSetId and SecurityProfile) + // to ensure the new region matches exactly. var encryption *armcompute.EncryptionImages for _, existing := range version.Properties.PublishingProfile.TargetRegions { if existing.Encryption != nil { @@ -654,21 +656,6 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, } } - // For ConfidentialVM images, Azure only allows cross-region replication when - // the encryption type is NonPersistedTPM. Detect CVM images by checking if - // any existing target region has a CVM security profile, and set NonPersistedTPM - // only on the new region (existing regions' encryption cannot be changed). - isCVM := encryption != nil && encryption.OSDiskImage != nil && encryption.OSDiskImage.SecurityProfile != nil - if isCVM { - encryption = &armcompute.EncryptionImages{ - OSDiskImage: &armcompute.OSDiskImageEncryption{ - SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ - ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), - }, - }, - } - } - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), From 593fec2d81b1e5f181a6a5434ac7b01c71c801e5 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Mon, 20 Apr 2026 22:37:55 +0000 Subject: [PATCH 06/21] fix cvm --- e2e/config/azure.go | 35 +++++++++++++++++++---------------- e2e/config/vhd.go | 12 +++++++----- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 32fbd87f08a..70b3de4bbe2 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -643,25 +643,28 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, return fmt.Errorf("create a new images client: %v", err) } - // Copy encryption settings from an existing target region if present. - // This is required for CVM images where Azure mandates that all target - // regions have matching encryption configuration. We copy the existing - // encryption verbatim (including DiskEncryptionSetId and SecurityProfile) - // to ensure the new region matches exactly. - var encryption *armcompute.EncryptionImages - for _, existing := range version.Properties.PublishingProfile.TargetRegions { - if existing.Encryption != nil { - encryption = existing.Encryption - break - } - } - - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ + newRegion := &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - Encryption: encryption, - }) + } + + if image.ConfidentialVM { + // For CVM images, Azure requires NonPersistedTPM encryption on all non-home + // target regions. Existing regions may lack this (legacy replication), and their + // encryption can't be changed (PropertyChangeNotAllowed). To work around this, + // replace all target regions with just the region we need, using NonPersistedTPM. + newRegion.Encryption = &armcompute.EncryptionImages{ + OSDiskImage: &armcompute.OSDiskImageEncryption{ + SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ + ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), + }, + }, + } + version.Properties.PublishingProfile.TargetRegions = []*armcompute.TargetRegion{newRegion} + } else { + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, newRegion) + } resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) if err != nil { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index d0492de6500..0338005c206 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -106,11 +106,12 @@ var ( Gallery: imageGalleryLinux, } VHDAzureLinuxV3CVMGen2 = &Image{ - Name: "AzureLinuxV3gen2CVM", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3CVMGen2, - Gallery: imageGalleryLinux, + Name: "AzureLinuxV3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: imageGalleryLinux, + ConfidentialVM: true, } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", @@ -307,6 +308,7 @@ type Image struct { UnsupportedGen2 bool IgnoreFailedCgroupTelemetryServices bool Flatcar bool + ConfidentialVM bool // OSDiskSizeGB overrides the default OS disk size (50 GB) when set. OSDiskSizeGB int32 } From 828af28ec4cefd7c65d42e3e49deef0708334398 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 01:05:49 +0000 Subject: [PATCH 07/21] fix cvm current region --- e2e/config/azure.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 70b3de4bbe2..3a28a7eff92 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -643,27 +643,34 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, return fmt.Errorf("create a new images client: %v", err) } - newRegion := &armcompute.TargetRegion{ - Name: &location, - RegionalReplicaCount: to.Ptr[int32](1), - StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - } - if image.ConfidentialVM { - // For CVM images, Azure requires NonPersistedTPM encryption on all non-home - // target regions. Existing regions may lack this (legacy replication), and their - // encryption can't be changed (PropertyChangeNotAllowed). To work around this, - // replace all target regions with just the region we need, using NonPersistedTPM. - newRegion.Encryption = &armcompute.EncryptionImages{ + // For CVM images, Azure requires NonPersistedTPM encryption on ALL target + // regions for cross-region replication. We must: + // 1. Keep all existing regions (can't drop the image version's home region) + // 2. Set NonPersistedTPM on every region (all must match) + // 3. Add the new region with NonPersistedTPM + cvmEncryption := &armcompute.EncryptionImages{ OSDiskImage: &armcompute.OSDiskImageEncryption{ SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), }, }, } - version.Properties.PublishingProfile.TargetRegions = []*armcompute.TargetRegion{newRegion} + for _, region := range version.Properties.PublishingProfile.TargetRegions { + region.Encryption = cvmEncryption + } + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ + Name: &location, + RegionalReplicaCount: to.Ptr[int32](1), + StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), + Encryption: cvmEncryption, + }) } else { - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, newRegion) + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ + Name: &location, + RegionalReplicaCount: to.Ptr[int32](1), + StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), + }) } resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) From 8f0ff2ed1afd73ea20688e81853fef8f2b5dbad1 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 13:00:06 +0000 Subject: [PATCH 08/21] fix cvm current region --- e2e/config/azure.go | 34 +++++----------------------------- e2e/config/vhd.go | 12 +++++------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 3a28a7eff92..aa7b09c8bfc 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -643,35 +643,11 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, return fmt.Errorf("create a new images client: %v", err) } - if image.ConfidentialVM { - // For CVM images, Azure requires NonPersistedTPM encryption on ALL target - // regions for cross-region replication. We must: - // 1. Keep all existing regions (can't drop the image version's home region) - // 2. Set NonPersistedTPM on every region (all must match) - // 3. Add the new region with NonPersistedTPM - cvmEncryption := &armcompute.EncryptionImages{ - OSDiskImage: &armcompute.OSDiskImageEncryption{ - SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ - ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), - }, - }, - } - for _, region := range version.Properties.PublishingProfile.TargetRegions { - region.Encryption = cvmEncryption - } - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ - Name: &location, - RegionalReplicaCount: to.Ptr[int32](1), - StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - Encryption: cvmEncryption, - }) - } else { - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ - Name: &location, - RegionalReplicaCount: to.Ptr[int32](1), - StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - }) - } + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ + Name: &location, + RegionalReplicaCount: to.Ptr[int32](1), + StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), + }) resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) if err != nil { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 0338005c206..d0492de6500 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -106,12 +106,11 @@ var ( Gallery: imageGalleryLinux, } VHDAzureLinuxV3CVMGen2 = &Image{ - Name: "AzureLinuxV3gen2CVM", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3CVMGen2, - Gallery: imageGalleryLinux, - ConfidentialVM: true, + Name: "AzureLinuxV3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: imageGalleryLinux, } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", @@ -308,7 +307,6 @@ type Image struct { UnsupportedGen2 bool IgnoreFailedCgroupTelemetryServices bool Flatcar bool - ConfidentialVM bool // OSDiskSizeGB overrides the default OS disk size (50 GB) when set. OSDiskSizeGB int32 } From 55495da2410628b6329fd7bba22d6d438440144a Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 13:48:13 +0000 Subject: [PATCH 09/21] log cvm encryption type --- e2e/config/azure.go | 26 +++++++++++++++++++++++--- e2e/config/vhd.go | 12 +++++++----- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index aa7b09c8bfc..a4a4b94fa0d 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -569,7 +569,17 @@ func (a *AzureClient) ensureReplication(ctx context.Context, image *Image, versi } regions := make([]string, 0, len(version.Properties.PublishingProfile.TargetRegions)) for _, targetRegion := range version.Properties.PublishingProfile.TargetRegions { - regions = append(regions, *targetRegion.Name) + regionInfo := *targetRegion.Name + if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil && targetRegion.Encryption.OSDiskImage.SecurityProfile != nil { + regionInfo += fmt.Sprintf(" (encryption: %v)", *targetRegion.Encryption.OSDiskImage.SecurityProfile.ConfidentialVMEncryptionType) + } else if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil { + regionInfo += " (encryption: osDiskImage set, no securityProfile)" + } else if targetRegion.Encryption != nil { + regionInfo += " (encryption: set, no osDiskImage)" + } else { + regionInfo += " (encryption: nil)" + } + regions = append(regions, regionInfo) } toolkit.Logf(ctx, "Replicating to region %s, available regions: %s, image version %s", location, strings.Join(regions, ", "), *version.ID) toolkit.Logf(ctx, "##vso[task.logissue type=warning;]Replicating to region %s", location) @@ -643,11 +653,21 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, return fmt.Errorf("create a new images client: %v", err) } - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ + newRegion := &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - }) + } + if image.ConfidentialVM { + newRegion.Encryption = &armcompute.EncryptionImages{ + OSDiskImage: &armcompute.OSDiskImageEncryption{ + SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ + ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), + }, + }, + } + } + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, newRegion) resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) if err != nil { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index d0492de6500..0338005c206 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -106,11 +106,12 @@ var ( Gallery: imageGalleryLinux, } VHDAzureLinuxV3CVMGen2 = &Image{ - Name: "AzureLinuxV3gen2CVM", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3CVMGen2, - Gallery: imageGalleryLinux, + Name: "AzureLinuxV3gen2CVM", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3CVMGen2, + Gallery: imageGalleryLinux, + ConfidentialVM: true, } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", @@ -307,6 +308,7 @@ type Image struct { UnsupportedGen2 bool IgnoreFailedCgroupTelemetryServices bool Flatcar bool + ConfidentialVM bool // OSDiskSizeGB overrides the default OS disk size (50 GB) when set. OSDiskSizeGB int32 } From 58efe01fe3b366cd9cee9f8dcb80bc5d3b620df2 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 14:27:05 +0000 Subject: [PATCH 10/21] copy encryption in e2e image replication --- e2e/config/azure.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index a4a4b94fa0d..8be001eaf2b 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -569,17 +569,7 @@ func (a *AzureClient) ensureReplication(ctx context.Context, image *Image, versi } regions := make([]string, 0, len(version.Properties.PublishingProfile.TargetRegions)) for _, targetRegion := range version.Properties.PublishingProfile.TargetRegions { - regionInfo := *targetRegion.Name - if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil && targetRegion.Encryption.OSDiskImage.SecurityProfile != nil { - regionInfo += fmt.Sprintf(" (encryption: %v)", *targetRegion.Encryption.OSDiskImage.SecurityProfile.ConfidentialVMEncryptionType) - } else if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil { - regionInfo += " (encryption: osDiskImage set, no securityProfile)" - } else if targetRegion.Encryption != nil { - regionInfo += " (encryption: set, no osDiskImage)" - } else { - regionInfo += " (encryption: nil)" - } - regions = append(regions, regionInfo) + regions = append(regions, *targetRegion.Name) } toolkit.Logf(ctx, "Replicating to region %s, available regions: %s, image version %s", location, strings.Join(regions, ", "), *version.ID) toolkit.Logf(ctx, "##vso[task.logissue type=warning;]Replicating to region %s", location) @@ -662,7 +652,7 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, newRegion.Encryption = &armcompute.EncryptionImages{ OSDiskImage: &armcompute.OSDiskImageEncryption{ SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ - ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeNonPersistedTPM), + ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeEncryptedVMGuestStateOnlyWithPmk), }, }, } @@ -728,6 +718,7 @@ func DefaultRetryOpts() policy.RetryOptions { http.StatusBadGateway, // 502 http.StatusServiceUnavailable, // 503 http.StatusGatewayTimeout, // 504 + http.StatusNotFound, // 404 }, } } From 1ad888f4981412fa0d6661df38508640e53c47bd Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 14:30:38 +0000 Subject: [PATCH 11/21] fix conflict --- e2e/config/azure.go | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 8be001eaf2b..dd5444766e9 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -718,7 +718,6 @@ func DefaultRetryOpts() policy.RetryOptions { http.StatusBadGateway, // 502 http.StatusServiceUnavailable, // 503 http.StatusGatewayTimeout, // 504 - http.StatusNotFound, // 404 }, } } From bcd7a6ef2fae779b05eb364b766ccdf4e12f05e1 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 17:38:28 +0000 Subject: [PATCH 12/21] try ubuntu --- e2e/config/vhd.go | 16 ++++++++-------- e2e/scenario_test.go | 12 ++++++------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 0338005c206..ea83ff223bf 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -105,14 +105,6 @@ var ( Distro: datamodel.AKSAzureLinuxV3Gen2, Gallery: imageGalleryLinux, } - VHDAzureLinuxV3CVMGen2 = &Image{ - Name: "AzureLinuxV3gen2CVM", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3CVMGen2, - Gallery: imageGalleryLinux, - ConfidentialVM: true, - } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, @@ -176,6 +168,14 @@ var ( Distro: datamodel.AKSUbuntuContainerd2404Gen2, Gallery: imageGalleryLinux, } + VHDUbuntu2404CVMGen2Containerd = &Image{ + Name: "2404gen2CVMcontainerd", + OS: OSUbuntu, + Arch: "amd64", + Distro: datamodel.AKSUbuntuContainerd2404CVMGen2, + Gallery: imageGalleryLinux, + ConfidentialVM: true, + } VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 0ac1bd0dee0..46b07adeedb 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1422,12 +1422,12 @@ func Test_Ubuntu2204_ArtifactStreaming_FIPS_Scriptless(t *testing.T) { }) } -func Test_AzureLinuxV3_ArtifactStreaming_CVM(t *testing.T) { +func Test_Ubuntu2404_ArtifactStreaming_CVM(t *testing.T) { RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 CVM node using artifact streaming can be properly bootstrapped", + Description: "tests that a new ubuntu 2404 CVM node using artifact streaming can be properly bootstrapped", Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3CVMGen2, + VHD: config.VHDUbuntu2404CVMGen2Containerd, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { nbc.EnableArtifactStreaming = true nbc.AgentPoolProfile.VMSize = "Standard_DC2as_v5" @@ -1447,15 +1447,15 @@ func Test_AzureLinuxV3_ArtifactStreaming_CVM(t *testing.T) { }) } -func Test_AzureLinuxV3_ArtifactStreaming_CVM_Scriptless(t *testing.T) { +func Test_Ubuntu2404_ArtifactStreaming_CVM_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 CVM node using artifact streaming can be properly bootstrapped", + Description: "tests that a new ubuntu 2404 CVM node using artifact streaming can be properly bootstrapped", Tags: Tags{ Scriptless: true, }, Config: Config{ Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3CVMGen2, + VHD: config.VHDUbuntu2404CVMGen2Containerd, AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { config.EnableArtifactStreaming = true config.VmSize = "Standard_DC2as_v5" From aabcb6418ead1f26d9b74e95c581567c1bb6a463 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 17:40:10 +0000 Subject: [PATCH 13/21] ubuntu log dump --- e2e/config/azure.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index dd5444766e9..5b757d3cc2e 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -569,7 +569,17 @@ func (a *AzureClient) ensureReplication(ctx context.Context, image *Image, versi } regions := make([]string, 0, len(version.Properties.PublishingProfile.TargetRegions)) for _, targetRegion := range version.Properties.PublishingProfile.TargetRegions { - regions = append(regions, *targetRegion.Name) + regionInfo := *targetRegion.Name + if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil && targetRegion.Encryption.OSDiskImage.SecurityProfile != nil { + regionInfo += fmt.Sprintf(" (encryption: %v)", *targetRegion.Encryption.OSDiskImage.SecurityProfile.ConfidentialVMEncryptionType) + } else if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil { + regionInfo += " (encryption: osDiskImage set, no securityProfile)" + } else if targetRegion.Encryption != nil { + regionInfo += " (encryption: set, no osDiskImage)" + } else { + regionInfo += " (encryption: nil)" + } + regions = append(regions, regionInfo) } toolkit.Logf(ctx, "Replicating to region %s, available regions: %s, image version %s", location, strings.Join(regions, ", "), *version.ID) toolkit.Logf(ctx, "##vso[task.logissue type=warning;]Replicating to region %s", location) From d4637c21214c9e6a088799ac70d95f28e91fd581 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 18:16:29 +0000 Subject: [PATCH 14/21] remove cvm --- e2e/config/azure.go | 26 +++------------------- e2e/config/vhd.go | 10 --------- e2e/scenario_test.go | 53 -------------------------------------------- e2e/test_helpers.go | 36 ------------------------------ 4 files changed, 3 insertions(+), 122 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 5b757d3cc2e..aa7b09c8bfc 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -569,17 +569,7 @@ func (a *AzureClient) ensureReplication(ctx context.Context, image *Image, versi } regions := make([]string, 0, len(version.Properties.PublishingProfile.TargetRegions)) for _, targetRegion := range version.Properties.PublishingProfile.TargetRegions { - regionInfo := *targetRegion.Name - if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil && targetRegion.Encryption.OSDiskImage.SecurityProfile != nil { - regionInfo += fmt.Sprintf(" (encryption: %v)", *targetRegion.Encryption.OSDiskImage.SecurityProfile.ConfidentialVMEncryptionType) - } else if targetRegion.Encryption != nil && targetRegion.Encryption.OSDiskImage != nil { - regionInfo += " (encryption: osDiskImage set, no securityProfile)" - } else if targetRegion.Encryption != nil { - regionInfo += " (encryption: set, no osDiskImage)" - } else { - regionInfo += " (encryption: nil)" - } - regions = append(regions, regionInfo) + regions = append(regions, *targetRegion.Name) } toolkit.Logf(ctx, "Replicating to region %s, available regions: %s, image version %s", location, strings.Join(regions, ", "), *version.ID) toolkit.Logf(ctx, "##vso[task.logissue type=warning;]Replicating to region %s", location) @@ -653,21 +643,11 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, return fmt.Errorf("create a new images client: %v", err) } - newRegion := &armcompute.TargetRegion{ + version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), StorageAccountType: to.Ptr(armcompute.StorageAccountTypeStandardLRS), - } - if image.ConfidentialVM { - newRegion.Encryption = &armcompute.EncryptionImages{ - OSDiskImage: &armcompute.OSDiskImageEncryption{ - SecurityProfile: &armcompute.OSDiskImageSecurityProfile{ - ConfidentialVMEncryptionType: to.Ptr(armcompute.ConfidentialVMEncryptionTypeEncryptedVMGuestStateOnlyWithPmk), - }, - }, - } - } - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, newRegion) + }) resp, err := galleryImageVersion.BeginCreateOrUpdate(ctx, image.Gallery.ResourceGroupName, image.Gallery.Name, image.Name, *version.Name, *version, nil) if err != nil { diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index ea83ff223bf..1d15887f83f 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -168,15 +168,6 @@ var ( Distro: datamodel.AKSUbuntuContainerd2404Gen2, Gallery: imageGalleryLinux, } - VHDUbuntu2404CVMGen2Containerd = &Image{ - Name: "2404gen2CVMcontainerd", - OS: OSUbuntu, - Arch: "amd64", - Distro: datamodel.AKSUbuntuContainerd2404CVMGen2, - Gallery: imageGalleryLinux, - ConfidentialVM: true, - } - VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, @@ -308,7 +299,6 @@ type Image struct { UnsupportedGen2 bool IgnoreFailedCgroupTelemetryServices bool Flatcar bool - ConfidentialVM bool // OSDiskSizeGB overrides the default OS disk size (50 GB) when set. OSDiskSizeGB int32 } diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 46b07adeedb..54a8d74826b 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1422,59 +1422,6 @@ func Test_Ubuntu2204_ArtifactStreaming_FIPS_Scriptless(t *testing.T) { }) } -func Test_Ubuntu2404_ArtifactStreaming_CVM(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new ubuntu 2404 CVM node using artifact streaming can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDUbuntu2404CVMGen2Containerd, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.EnableArtifactStreaming = true - nbc.AgentPoolProfile.VMSize = "Standard_DC2as_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_DC2as_v5") - vmss.Properties = addConfidentialVMToVMSS(vmss.Properties) - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - }, - }, - }) -} - -func Test_Ubuntu2404_ArtifactStreaming_CVM_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new ubuntu 2404 CVM node using artifact streaming can be properly bootstrapped", - Tags: Tags{ - Scriptless: true, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDUbuntu2404CVMGen2Containerd, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.EnableArtifactStreaming = true - config.VmSize = "Standard_DC2as_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_DC2as_v5") - vmss.Properties = addConfidentialVMToVMSS(vmss.Properties) - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - }, - }, - }) -} - func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations", diff --git a/e2e/test_helpers.go b/e2e/test_helpers.go index 753e27fb558..71312b8b108 100644 --- a/e2e/test_helpers.go +++ b/e2e/test_helpers.go @@ -548,42 +548,6 @@ func addTrustedLaunchToVMSS(properties *armcompute.VirtualMachineScaleSetPropert return properties } -func addConfidentialVMToVMSS(properties *armcompute.VirtualMachineScaleSetProperties) *armcompute.VirtualMachineScaleSetProperties { - if properties == nil { - properties = &armcompute.VirtualMachineScaleSetProperties{} - } - - if properties.VirtualMachineProfile == nil { - properties.VirtualMachineProfile = &armcompute.VirtualMachineScaleSetVMProfile{} - } - - if properties.VirtualMachineProfile.SecurityProfile == nil { - properties.VirtualMachineProfile.SecurityProfile = &armcompute.SecurityProfile{} - } - - properties.VirtualMachineProfile.SecurityProfile.SecurityType = to.Ptr(armcompute.SecurityTypesConfidentialVM) - if properties.VirtualMachineProfile.SecurityProfile.UefiSettings == nil { - properties.VirtualMachineProfile.SecurityProfile.UefiSettings = &armcompute.UefiSettings{} - } - properties.VirtualMachineProfile.SecurityProfile.UefiSettings.SecureBootEnabled = to.Ptr(true) - properties.VirtualMachineProfile.SecurityProfile.UefiSettings.VTpmEnabled = to.Ptr(true) - - if properties.VirtualMachineProfile.StorageProfile == nil { - properties.VirtualMachineProfile.StorageProfile = &armcompute.VirtualMachineScaleSetStorageProfile{} - } - if properties.VirtualMachineProfile.StorageProfile.OSDisk == nil { - properties.VirtualMachineProfile.StorageProfile.OSDisk = &armcompute.VirtualMachineScaleSetOSDisk{} - } - if properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk == nil { - properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk = &armcompute.VirtualMachineScaleSetManagedDiskParameters{} - } - properties.VirtualMachineProfile.StorageProfile.OSDisk.ManagedDisk.SecurityProfile = &armcompute.VMDiskSecurityProfile{ - SecurityEncryptionType: to.Ptr(armcompute.SecurityEncryptionTypesVMGuestStateOnly), - } - - return properties -} - func createVMExtensionLinuxAKSNode(ctx context.Context, location *string) (*armcompute.VirtualMachineScaleSetExtension, error) { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() From 0a377309b0fb41db25de8eea1abf5afca610c5bd Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 18:58:28 +0000 Subject: [PATCH 15/21] test only kata no AS --- e2e/config/vhd.go | 7 +++++++ e2e/scenario_test.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 1d15887f83f..5f3dd60708c 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -105,6 +105,13 @@ var ( Distro: datamodel.AKSAzureLinuxV3Gen2, Gallery: imageGalleryLinux, } + VHDAzureLinuxV3KataGen2 = &Image{ + Name: "AzureLinuxV3katagen2", + OS: OSAzureLinux, + Arch: "amd64", + Distro: datamodel.AKSAzureLinuxV3Gen2Kata, + Gallery: imageGalleryLinux, + } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 54a8d74826b..340d24bf884 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1235,6 +1235,54 @@ func Test_AzureLinuxV3_ArtifactStreaming_Scriptless(t *testing.T) { }) } +func Test_AzureLinuxV3_PodSandboxing(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) can be properly bootstrapped without artifact streaming", + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3KataGen2, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.AgentPoolProfile.VMSize = "Standard_D4s_v3" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") + ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") + ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") + }, + }, + }) +} + +func Test_AzureLinuxV3_PodSandboxing_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) can be properly bootstrapped without artifact streaming", + Tags: Tags{ + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterKubenet, + VHD: config.VHDAzureLinuxV3KataGen2, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.IsKata = true + config.VmSize = "Standard_D4s_v3" + }, + VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { + vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") + ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") + ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") + }, + }, + }) +} + func Test_Ubuntu2204_ArtifactStreaming_ARM64_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Description: "tests that a new ubuntu 2204 node using artifact streaming and ARM64 architecture can be properly bootstrapped", From f9d47ee5a4e9cfeda9c3fff23efe9eba90bc5bfc Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Tue, 21 Apr 2026 19:39:04 +0000 Subject: [PATCH 16/21] test only kata no AS, fix disk --- e2e/scenario_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 340d24bf884..082c3e1f10f 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1246,6 +1246,7 @@ func Test_AzureLinuxV3_PodSandboxing(t *testing.T) { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) }, Validator: func(ctx context.Context, s *Scenario) { ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") @@ -1272,6 +1273,7 @@ func Test_AzureLinuxV3_PodSandboxing_Scriptless(t *testing.T) { }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) }, Validator: func(ctx context.Context, s *Scenario) { ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") From 76cfb46686e960aad820dcdbef08b9ae7b94a86a Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Wed, 22 Apr 2026 14:58:23 +0000 Subject: [PATCH 17/21] restore kata to kata+AS e2e --- e2e/scenario_test.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 082c3e1f10f..8b5b9153276 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1235,20 +1235,25 @@ func Test_AzureLinuxV3_ArtifactStreaming_Scriptless(t *testing.T) { }) } -func Test_AzureLinuxV3_PodSandboxing(t *testing.T) { +func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing(t *testing.T) { RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) can be properly bootstrapped without artifact streaming", + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", Config: Config{ Cluster: ClusterKubenet, VHD: config.VHDAzureLinuxV3KataGen2, BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.AgentPoolProfile.VMSize = "Standard_D4s_v3" + nbc.EnableArtifactStreaming = true + nbc.AgentPoolProfile.VMSize = "Standard_D4ds_v5" }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) }, Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") @@ -1258,9 +1263,9 @@ func Test_AzureLinuxV3_PodSandboxing(t *testing.T) { }) } -func Test_AzureLinuxV3_PodSandboxing_Scriptless(t *testing.T) { +func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) can be properly bootstrapped without artifact streaming", + Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", Tags: Tags{ Scriptless: true, }, @@ -1268,14 +1273,19 @@ func Test_AzureLinuxV3_PodSandboxing_Scriptless(t *testing.T) { Cluster: ClusterKubenet, VHD: config.VHDAzureLinuxV3KataGen2, AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true config.IsKata = true - config.VmSize = "Standard_D4s_v3" + config.VmSize = "Standard_D4ds_v5" }, VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4s_v3") + vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) }, Validator: func(ctx context.Context, s *Scenario) { + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") From 414434192f7449a1c1a2edfc998d2f8f7def7166 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Wed, 22 Apr 2026 19:12:54 +0000 Subject: [PATCH 18/21] moved kata e2es to Harsha's branch --- e2e/config/vhd.go | 7 ------ e2e/scenario_test.go | 60 -------------------------------------------- 2 files changed, 67 deletions(-) diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index 5f3dd60708c..1d15887f83f 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -105,13 +105,6 @@ var ( Distro: datamodel.AKSAzureLinuxV3Gen2, Gallery: imageGalleryLinux, } - VHDAzureLinuxV3KataGen2 = &Image{ - Name: "AzureLinuxV3katagen2", - OS: OSAzureLinux, - Arch: "amd64", - Distro: datamodel.AKSAzureLinuxV3Gen2Kata, - Gallery: imageGalleryLinux, - } VHDAzureLinux3OSGuard = &Image{ Name: "AzureLinuxOSGuardOSGuardV3gen2fipsTL", OS: OSAzureLinux, diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 8b5b9153276..54a8d74826b 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1235,66 +1235,6 @@ func Test_AzureLinuxV3_ArtifactStreaming_Scriptless(t *testing.T) { }) } -func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3KataGen2, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.EnableArtifactStreaming = true - nbc.AgentPoolProfile.VMSize = "Standard_D4ds_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") - vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") - ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") - ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") - }, - }, - }) -} - -func Test_AzureLinuxV3_ArtifactStreaming_PodSandboxing_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new azure linux v3 node with pod sandboxing (Kata) using artifact streaming can be properly bootstrapped", - Tags: Tags{ - Scriptless: true, - }, - Config: Config{ - Cluster: ClusterKubenet, - VHD: config.VHDAzureLinuxV3KataGen2, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.EnableArtifactStreaming = true - config.IsKata = true - config.VmSize = "Standard_D4ds_v5" - }, - VMConfigMutator: func(vmss *armcompute.VirtualMachineScaleSet) { - vmss.SKU.Name = to.Ptr("Standard_D4ds_v5") - vmss.Properties.VirtualMachineProfile.StorageProfile.OSDisk.DiffDiskSettings.Placement = to.Ptr(armcompute.DiffDiskPlacementCacheDisk) - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - ValidateFileExists(ctx, s, "/usr/bin/kata-runtime") - ValidateFileExists(ctx, s, "/usr/bin/containerd-shim-kata-v2") - ValidateFileHasContent(ctx, s, "/etc/containerd/config.toml", "containerd.runtimes.kata") - }, - }, - }) -} - func Test_Ubuntu2204_ArtifactStreaming_ARM64_Scriptless(t *testing.T) { RunScenario(t, &Scenario{ Description: "tests that a new ubuntu 2204 node using artifact streaming and ARM64 architecture can be properly bootstrapped", From c1a6fc98937e8ea4d92c63568534c6de19e07f9d Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Thu, 23 Apr 2026 21:01:00 +0000 Subject: [PATCH 19/21] add network isolated cluster tests --- e2e/scenario_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 92866d83cf6..589ba00f847 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1458,6 +1458,89 @@ func Test_Ubuntu2204_ArtifactStreaming_FIPS_Scriptless(t *testing.T) { }) } +func Test_Ubuntu2204_ArtifactStreaming_NetworkIsolatedCluster(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 node in a network isolated cluster using artifact streaming can be properly bootstrapped", + Tags: Tags{ + NetworkIsolated: true, + NonAnonymousACR: true, + }, + Config: Config{ + Cluster: ClusterAzureNetworkIsolated, + VHD: config.VHDUbuntu2204Gen2Containerd, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.EnableArtifactStreaming = true + nbc.OutboundType = datamodel.OutboundTypeBlock + nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ + PrivateEgress: &datamodel.PrivateEgress{ + Enabled: true, + ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)), + }, + } + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true + nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true + nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf( + "https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz", + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion) + nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml" + nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider" + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + +func Test_Ubuntu2204_ArtifactStreaming_NetworkIsolatedCluster_Scriptless(t *testing.T) { + RunScenario(t, &Scenario{ + Description: "tests that a new ubuntu 2204 node in a network isolated cluster using artifact streaming can be properly bootstrapped", + Tags: Tags{ + NetworkIsolated: true, + NonAnonymousACR: true, + Scriptless: true, + }, + Config: Config{ + Cluster: ClusterAzureNetworkIsolated, + VHD: config.VHDUbuntu2204Gen2Containerd, + BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { + nbc.OutboundType = datamodel.OutboundTypeBlock + nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ + PrivateEgress: &datamodel.PrivateEgress{ + Enabled: true, + ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)), + }, + } + nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true + nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true + nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf( + "https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz", + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, + nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion) + nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml" + nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider" + }, + AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { + config.EnableArtifactStreaming = true + }, + Validator: func(ctx context.Context, s *Scenario) { + ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) + ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") + ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") + ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") + ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") + }, + }, + }) +} + func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations", From 126b8473241fc23b7addfef48c7ceb107666d388 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Thu, 23 Apr 2026 21:07:49 +0000 Subject: [PATCH 20/21] cleanup pr --- e2e/config/azure.go | 1 - e2e/config/vhd.go | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index aa7b09c8bfc..d0de6f04619 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -642,7 +642,6 @@ func (a *AzureClient) replicateImageVersionToCurrentRegion(ctx context.Context, if err != nil { return fmt.Errorf("create a new images client: %v", err) } - version.Properties.PublishingProfile.TargetRegions = append(version.Properties.PublishingProfile.TargetRegions, &armcompute.TargetRegion{ Name: &location, RegionalReplicaCount: to.Ptr[int32](1), diff --git a/e2e/config/vhd.go b/e2e/config/vhd.go index d0556540550..f23b1430a0f 100644 --- a/e2e/config/vhd.go +++ b/e2e/config/vhd.go @@ -180,6 +180,7 @@ var ( Distro: datamodel.AKSUbuntuContainerd2404Gen2, Gallery: imageGalleryLinux, } + VHDUbuntu2404ArmContainerd = &Image{ Name: "2404gen2arm64containerd", OS: OSUbuntu, From 6f87ce34665a483b3316c39df0aa02bb423f19c5 Mon Sep 17 00:00:00 2001 From: Mark Ibrahim Date: Fri, 24 Apr 2026 13:47:32 +0000 Subject: [PATCH 21/21] remove netiso --- e2e/scenario_test.go | 43 ------------------------------------------- 1 file changed, 43 deletions(-) diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go index 589ba00f847..bcf94c5e585 100644 --- a/e2e/scenario_test.go +++ b/e2e/scenario_test.go @@ -1498,49 +1498,6 @@ func Test_Ubuntu2204_ArtifactStreaming_NetworkIsolatedCluster(t *testing.T) { }) } -func Test_Ubuntu2204_ArtifactStreaming_NetworkIsolatedCluster_Scriptless(t *testing.T) { - RunScenario(t, &Scenario{ - Description: "tests that a new ubuntu 2204 node in a network isolated cluster using artifact streaming can be properly bootstrapped", - Tags: Tags{ - NetworkIsolated: true, - NonAnonymousACR: true, - Scriptless: true, - }, - Config: Config{ - Cluster: ClusterAzureNetworkIsolated, - VHD: config.VHDUbuntu2204Gen2Containerd, - BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) { - nbc.OutboundType = datamodel.OutboundTypeBlock - nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{ - PrivateEgress: &datamodel.PrivateEgress{ - Enabled: true, - ContainerRegistryServer: fmt.Sprintf("%s.azurecr.io/aks-managed-repository", config.PrivateACRNameNotAnon(config.Config.DefaultLocation)), - }, - } - nbc.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity = true - nbc.AgentPoolProfile.KubernetesConfig.UseManagedIdentity = true - nbc.K8sComponents.LinuxCredentialProviderURL = fmt.Sprintf( - "https://packages.aks.azure.com/cloud-provider-azure/v%s/binaries/azure-acr-credential-provider-linux-amd64-v%s.tar.gz", - nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion, - nbc.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion) - nbc.KubeletConfig["--image-credential-provider-config"] = "/var/lib/kubelet/credential-provider-config.yaml" - nbc.KubeletConfig["--image-credential-provider-bin-dir"] = "/var/lib/kubelet/credential-provider" - }, - AKSNodeConfigMutator: func(config *aksnodeconfigv1.Configuration) { - config.EnableArtifactStreaming = true - }, - Validator: func(ctx context.Context, s *Scenario) { - ValidateDirectoryContent(ctx, s, "/opt/azure", []string{"outbound-check-skipped"}) - ValidateNonEmptyDirectory(ctx, s, "/etc/overlaybd") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-snapshotter.service") - ValidateSystemdUnitIsRunning(ctx, s, "overlaybd-tcmu.service") - ValidateSystemdUnitIsRunning(ctx, s, "acr-mirror.service") - ValidateSystemdUnitIsRunning(ctx, s, "containerd.service") - }, - }, - }) -} - func Test_Ubuntu2204_ChronyRestarts_Taints_And_Tolerations(t *testing.T) { RunScenario(t, &Scenario{ Description: "Tests that the chrony service restarts if it is killed. Also tests taints and tolerations",