CORS-4072: dualstack support for AWS

pkg/asset/installconfig/aws/route53.go

@@ -191,39 +191,59 @@ type CreateRecordInput struct {
 	ZoneID string
 	// ID of the Hosted Zone for Alias record.
 	AliasZoneID string
+	// Whether to also include an AAAA record.
+	EnableAAAA bool
 }
 
 // CreateOrUpdateRecord Creates or Updates the Route53 Record for the cluster endpoint.
 func (c *Route53Client) CreateOrUpdateRecord(ctx context.Context, in *CreateRecordInput) error {
-	recordSet := &route53types.ResourceRecordSet{
-		Name: aws.String(in.Name),
-	}
+	recordSets := []*route53types.ResourceRecordSet{}
+
 	if cnameRegions.Has(in.Region) {
-		recordSet.Type = route53types.RRTypeCname
-		recordSet.TTL = aws.Int64(10)
-		recordSet.ResourceRecords = []route53types.ResourceRecord{
-			{Value: aws.String(in.DNSTarget)},
-		}
+		recordSets = append(recordSets, &route53types.ResourceRecordSet{
+			Name: aws.String(in.Name),
+			Type: route53types.RRTypeCname,
+			TTL:  aws.Int64(10),
+			ResourceRecords: []route53types.ResourceRecord{
+				{Value: aws.String(in.DNSTarget)},
+			},
+		})
 	} else {
-		recordSet.Type = route53types.RRTypeA
-		recordSet.AliasTarget = &route53types.AliasTarget{
-			DNSName:              aws.String(in.DNSTarget),
-			HostedZoneId:         aws.String(in.AliasZoneID),
-			EvaluateTargetHealth: false,
+		recordSets = append(recordSets, &route53types.ResourceRecordSet{
+			Name: aws.String(in.Name),
+			Type: route53types.RRTypeA,
+			AliasTarget: &route53types.AliasTarget{
+				DNSName:              aws.String(in.DNSTarget),
+				HostedZoneId:         aws.String(in.AliasZoneID),
+				EvaluateTargetHealth: false,
+			},
+		})
+
+		if in.EnableAAAA {
+			recordSets = append(recordSets, &route53types.ResourceRecordSet{
+				Name: aws.String(in.Name),
+				Type: route53types.RRTypeAaaa,
+				AliasTarget: &route53types.AliasTarget{
+					DNSName:              aws.String(in.DNSTarget),
+					HostedZoneId:         aws.String(in.AliasZoneID),
+					EvaluateTargetHealth: false,
+				},
+			})
 		}
 	}
+
 	input := &route53.ChangeResourceRecordSetsInput{
 		HostedZoneId: aws.String(in.ZoneID),
 		ChangeBatch: &route53types.ChangeBatch{
 			Comment: aws.String(fmt.Sprintf("Creating record %s", in.Name)),
-			Changes: []route53types.Change{
-				{
-					Action:            route53types.ChangeActionUpsert,
-					ResourceRecordSet: recordSet,
-				},
-			},
 		},
 	}
+	for _, recordSet := range recordSets {
+		input.ChangeBatch.Changes = append(input.ChangeBatch.Changes, route53types.Change{
+			Action:            route53types.ChangeActionUpsert,
+			ResourceRecordSet: recordSet,
+		})
+	}
 
 	_, err := c.client.ChangeResourceRecordSets(ctx, input)
 

pkg/asset/machines/aws/awsmachines.go

@@ -22,6 +22,7 @@ import (
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
+	"github.com/openshift/installer/pkg/types/network"
 )
 
 // MachineInput defines the inputs needed to generate a machine asset.
@@ -32,6 +33,7 @@ type MachineInput struct {
 	Tags           capa.Tags
 	PublicIP       bool
 	PublicIpv4Pool string
+	IPFamily       network.IPFamily
 	Ignition       *capa.Ignition
 }
 
@@ -122,6 +124,25 @@ func GenerateMachines(clusterID string, in *MachineInput) ([]*asset.RuntimeFile,
 			awsMachine.Spec.RootVolume.Throughput = ptr.To(int64(*throughput))
 		}
 
+		if in.IPFamily.DualStackEnabled() {
+			awsMachine.Spec.PrivateDNSName = &capa.PrivateDNSName{
+				EnableResourceNameDNSAAAARecord: ptr.To(true),
+				EnableResourceNameDNSARecord:    ptr.To(true),
+				// Only resource-name supports A and AAAA records for private host names
+				// See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/hostname-types.html#ec2-instance-private-hostnames
+				HostnameType: ptr.To("resource-name"),
+			}
+			awsMachine.Spec.InstanceMetadataOptions.HTTPProtocolIPv6 = capa.InstanceMetadataEndpointStateEnabled
+
+			// AssignPrimaryIPv6 is required for IPv6 primary to register instances to IPv6 target groups
+			switch in.IPFamily {
+			case network.DualStackIPv6Primary:
+				awsMachine.Spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateEnabled)
+			case network.DualStackIPv4Primary:
+				awsMachine.Spec.AssignPrimaryIPv6 = ptr.To(capa.PrimaryIPv6AssignmentStateDisabled)
+			}
+		}
+
 		if in.Role == "bootstrap" {
 			awsMachine.Name = capiutils.GenerateBoostrapMachineName(clusterID)
 			awsMachine.Labels["install.openshift.io/bootstrap"] = ""

pkg/asset/machines/clusterapi.go

@@ -178,6 +178,7 @@ func (c *ClusterAPI) Generate(ctx context.Context, dependencies asset.Parents) e
 			Subnets:  subnets,
 			Tags:     tags,
 			PublicIP: publicOnlySubnets,
+			IPFamily: ic.AWS.IPFamily,
 			Ignition: &v1beta2.Ignition{
 				Version: "3.2",
 				// master machines should get ignition from the MCS on the bootstrap node
@@ -212,6 +213,7 @@ func (c *ClusterAPI) Generate(ctx context.Context, dependencies asset.Parents) e
 			Subnets:        bootstrapSubnets,
 			Pool:           &pool,
 			Tags:           tags,
+			IPFamily:       ic.AWS.IPFamily,
 			PublicIP:       publicOnlySubnets || (installConfig.Config.Publish == types.ExternalPublishingStrategy),
 			PublicIpv4Pool: ic.Platform.AWS.PublicIpv4Pool,
 			Ignition:       ignition,

pkg/asset/manifests/aws/cluster.go

@@ -14,6 +14,8 @@ import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/asset/machines/aws"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
+	"github.com/openshift/installer/pkg/ipnet"
+	"github.com/openshift/installer/pkg/types/network"
 )
 
 // BootstrapSSHDescription is the description for the
@@ -24,24 +26,32 @@ const BootstrapSSHDescription = "Bootstrap SSH Access"
 // GenerateClusterAssets generates the manifests for the cluster-api.
 func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installconfig.ClusterID) (*capiutils.GenerateClusterAssetsOutput, error) {
 	manifests := []*asset.RuntimeFile{}
+	platformAWS := ic.Config.AWS
+	enableIPv6 := platformAWS.IPFamily.DualStackEnabled()
 
-	tags, err := aws.CapaTagsFromUserTags(clusterID.InfraID, ic.Config.AWS.UserTags)
+	tags, err := aws.CapaTagsFromUserTags(clusterID.InfraID, platformAWS.UserTags)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get user tags: %w", err)
 	}
 
-	sshRuleCidr := []string{"0.0.0.0/0"}
+	var sshRuleCidrs []ipnet.IPNet
 	if !ic.Config.PublicAPI() {
-		sshRuleCidr = []string{capiutils.CIDRFromInstallConfig(ic).String()}
+		sshRuleCidrs = capiutils.MachineCIDRsFromInstallConfig(ic)
+	} else {
+		sshRuleCidrs = []ipnet.IPNet{*capiutils.AnyIPv4CidrBlock}
+		if enableIPv6 {
+			sshRuleCidrs = append(sshRuleCidrs, *capiutils.AnyIPv6CidrBlock)
+		}
 	}
 
+	targetGroupIPType := GetTargetGroupIPType(platformAWS.IPFamily)
 	awsCluster := &capa.AWSCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      clusterID.InfraID,
 			Namespace: capiutils.Namespace,
 		},
 		Spec: capa.AWSClusterSpec{
-			Region: ic.Config.AWS.Region,
+			Region: platformAWS.Region,
 			NetworkSpec: capa.NetworkSpec{
 				CNI: &capa.CNISpec{
 					CNIIngressRules: capa.CNIIngressRules{
@@ -142,14 +152,16 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 						SourceSecurityGroupRoles: []capa.SecurityGroupRole{"controlplane", "node"},
 					},
 					{
-						Description: BootstrapSSHDescription,
-						Protocol:    capa.SecurityGroupProtocolTCP,
-						FromPort:    22,
-						ToPort:      22,
-						CidrBlocks:  sshRuleCidr,
+						Description:    BootstrapSSHDescription,
+						Protocol:       capa.SecurityGroupProtocolTCP,
+						FromPort:       22,
+						ToPort:         22,
+						CidrBlocks:     capiutils.CIDRsToString(capiutils.GetIPv4CIDRs(sshRuleCidrs)),
+						IPv6CidrBlocks: capiutils.CIDRsToString(capiutils.GetIPv6CIDRs(sshRuleCidrs)),
 					},
 				},
-				NodePortIngressRuleCidrBlocks: []string{capiutils.CIDRFromInstallConfig(ic).String()},
+				// If the installer provisions the VPC, VPC IPv6 CIDR is unknown at install time and added after infraReady
+				NodePortIngressRuleCidrBlocks: capiutils.CIDRsToString(capiutils.MachineCIDRsFromInstallConfig(ic)),
 			},
 			S3Bucket: &capa.S3Bucket{
 				Name:                    GetIgnitionBucketName(clusterID.InfraID),
@@ -168,6 +180,7 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 					ThresholdCount:          ptr.To[int64](2),
 					UnhealthyThresholdCount: ptr.To[int64](2),
 				},
+				TargetGroupIPType: targetGroupIPType,
 				AdditionalListeners: []capa.AdditionalListenerSpec{
 					{
 						Port:     22623,
@@ -181,6 +194,7 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 							ThresholdCount:          ptr.To[int64](2),
 							UnhealthyThresholdCount: ptr.To[int64](2),
 						},
+						TargetGroupIPType: targetGroupIPType,
 					},
 				},
 				IngressRules: []capa.IngressRule{
@@ -198,7 +212,20 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 	}
 	awsCluster.SetGroupVersionKind(capa.GroupVersion.WithKind("AWSCluster"))
 
+	// Create a ingress rule to allow acccess to the API LB.
+	apiLBIngressRule := capa.IngressRule{
+		Description: "Kubernetes API Server traffic",
+		Protocol:    capa.SecurityGroupProtocolTCP,
+		FromPort:    6443,
+		ToPort:      6443,
+		CidrBlocks:  []string{capiutils.AnyIPv4CidrBlock.String()},
+	}
+	if enableIPv6 {
+		apiLBIngressRule.IPv6CidrBlocks = []string{capiutils.AnyIPv6CidrBlock.String()}
+	}
+
 	if ic.Config.PublicAPI() {
+		apiLBIngressRule.Description = "Kubernetes API Server traffic for public access"
 		awsCluster.Spec.SecondaryControlPlaneLoadBalancer = &capa.AWSLoadBalancerSpec{
 			Name:                   ptr.To(clusterID.InfraID + "-ext"),
 			LoadBalancerType:       capa.LoadBalancerTypeNLB,
@@ -211,26 +238,13 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 				ThresholdCount:          ptr.To[int64](2),
 				UnhealthyThresholdCount: ptr.To[int64](2),
 			},
-			IngressRules: []capa.IngressRule{
-				{
-					Description: "Kubernetes API Server traffic for public access",
-					Protocol:    capa.SecurityGroupProtocolTCP,
-					FromPort:    6443,
-					ToPort:      6443,
-					CidrBlocks:  []string{"0.0.0.0/0"},
-				},
-			},
+			TargetGroupIPType: targetGroupIPType,
+			IngressRules:      []capa.IngressRule{apiLBIngressRule},
 		}
 	} else {
 		awsCluster.Spec.ControlPlaneLoadBalancer.IngressRules = append(
 			awsCluster.Spec.ControlPlaneLoadBalancer.IngressRules,
-			capa.IngressRule{
-				Description: "Kubernetes API Server traffic",
-				Protocol:    capa.SecurityGroupProtocolTCP,
-				FromPort:    6443,
-				ToPort:      6443,
-				CidrBlocks:  []string{"0.0.0.0/0"},
-			},
+			apiLBIngressRule,
 		)
 	}
 
@@ -292,3 +306,19 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 func GetIgnitionBucketName(infraID string) string {
 	return fmt.Sprintf("openshift-bootstrap-data-%s", infraID)
 }
+
+// GetTargetGroupIPType returns the ipType of the target group based on ipFamily.
+func GetTargetGroupIPType(ipFamily network.IPFamily) *capa.TargetGroupIPType {
+	var tgIPType capa.TargetGroupIPType
+	switch ipFamily {
+	case network.DualStackIPv6Primary:
+		tgIPType = capa.TargetGroupIPTypeIPv6
+	case network.DualStackIPv4Primary:
+		tgIPType = capa.TargetGroupIPTypeIPv4
+	default:
+		// Default to IPv4 if not specified or invalid
+		tgIPType = capa.TargetGroupIPTypeIPv4
+	}
+
+	return &tgIPType
+}

pkg/asset/manifests/aws/zones.go

@@ -174,9 +174,16 @@ func setSubnets(ctx context.Context, in *networkInput) error {
 // TODO: create support to mock AWS API calls in the unit tests, then the method
 // GatherSubnetsFromMetadata() can be added in setSubnetsBYOVPC.
 func setSubnetsBYOVPC(in *networkInput) error {
-	in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{
+	enableIPv6 := in.InstallConfig.Config.AWS.IPFamily.DualStackEnabled()
+	// dualstack: we don't need to configure all IPv6 configurations, for example, VPC or subnet IPv6 CIDRs
+	// as CAPA will query AWS API to fill them in
+	vpcSpec := capa.VPCSpec{
 		ID: in.Subnets.vpc,
 	}
+	if enableIPv6 {
+		vpcSpec.IPv6 = &capa.IPv6{}
+	}
+	in.Cluster.Spec.NetworkSpec.VPC = vpcSpec
 
 	// Skip adding private subnets if this is a public-only subnets install.
 	// We need to skip because the Installer is tricked into thinking the public subnets are also private and we would
@@ -239,14 +246,19 @@ func setSubnetsManagedVPC(in *networkInput) error {
 		return fmt.Errorf("failed to get availability zones: %w", err)
 	}
 
+	enableIPv6 := in.InstallConfig.Config.AWS.IPFamily.DualStackEnabled()
 	isPublishingExternal := in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy
 	allAvailabilityZones := out.GetAvailabilityZones()
 	allEdgeZones := out.GetEdgeZones()
 
 	mainCIDR := capiutils.CIDRFromInstallConfig(in.InstallConfig)
-	in.Cluster.Spec.NetworkSpec.VPC = capa.VPCSpec{
+	vpcSpec := capa.VPCSpec{
 		CidrBlock: mainCIDR.String(),
 	}
+	if enableIPv6 {
+		vpcSpec.IPv6 = &capa.IPv6{}
+	}
+	in.Cluster.Spec.NetworkSpec.VPC = vpcSpec
 
 	// Base subnets count considering only private zones, leaving one free block to allow
 	// future subnet expansions in Day-2.
@@ -299,6 +311,7 @@ func setSubnetsManagedVPC(in *networkInput) error {
 				CidrBlock:        privateCIDRs[idxCIDR].String(),
 				ID:               fmt.Sprintf("%s-subnet-private-%s", in.ClusterID.InfraID, zone),
 				IsPublic:         false,
+				IsIPv6:           enableIPv6,
 			})
 		}
 		if isPublishingExternal {
@@ -307,6 +320,7 @@ func setSubnetsManagedVPC(in *networkInput) error {
 				CidrBlock:        publicCIDRs[idxCIDR].String(),
 				ID:               fmt.Sprintf("%s-subnet-public-%s", in.ClusterID.InfraID, zone),
 				IsPublic:         true,
+				IsIPv6:           enableIPv6,
 			})
 		}
 	}
@@ -341,6 +355,7 @@ func setSubnetsManagedVPC(in *networkInput) error {
 	}
 
 	// Create subnets from zone pool with type local-zone or wavelength-zone (edge zones)
+	// Important: We do not support IPv6 networking (i.e. dualstack) for edge zones
 	for idxCIDR, zone := range allEdgeZones {
 		in.Cluster.Spec.NetworkSpec.Subnets = append(in.Cluster.Spec.NetworkSpec.Subnets, capa.SubnetSpec{
 			AvailabilityZone: zone,