From 4a3089e568519c49b061afd87a3cdb50e9bdd9d1 Mon Sep 17 00:00:00 2001 From: Akarsh Ellore Sreenath Date: Mon, 16 Mar 2026 18:02:15 +0530 Subject: [PATCH] Add flex CIDR allocator controller --- pkg/cloudprovider/providers/oci/ccm.go | 9 + .../providers/oci/flex_cidr_controller.go | 203 +++++++ .../providers/oci/instances_test.go | 12 + pkg/csi/driver/bv_controller_test.go | 16 + pkg/flexcidr/flexcidr.go | 497 ++++++++++++++++++ pkg/flexcidr/flexcidr_test.go | 430 +++++++++++++++ pkg/oci/client/client.go | 2 + pkg/oci/client/client_test.go | 8 + pkg/oci/client/networking.go | 84 ++- pkg/volume/provisioner/block/block_test.go | 17 + pkg/volume/provisioner/fss/fss_test.go | 8 + 11 files changed, 1277 insertions(+), 9 deletions(-) create mode 100644 pkg/cloudprovider/providers/oci/flex_cidr_controller.go create mode 100644 pkg/flexcidr/flexcidr.go create mode 100644 pkg/flexcidr/flexcidr_test.go diff --git a/pkg/cloudprovider/providers/oci/ccm.go b/pkg/cloudprovider/providers/oci/ccm.go index acb2a05d42..b3c32b85ce 100644 --- a/pkg/cloudprovider/providers/oci/ccm.go +++ b/pkg/cloudprovider/providers/oci/ccm.go @@ -191,6 +191,14 @@ func (cp *CloudProvider) Initialize(clientBuilder cloudprovider.ControllerClient cp.logger, cp.instanceCache, cp.client) + flexCIDRController := NewFlexCIDRController( + factory.Core().V1().Nodes(), + factory.Core().V1().Services(), + cp.kubeclient, + cp, + cp.logger, + cp.instanceCache, + cp.client) nodeInformer := factory.Core().V1().Nodes() go nodeInformer.Informer().Run(wait.NeverStop) @@ -202,6 +210,7 @@ func (cp *CloudProvider) Initialize(clientBuilder cloudprovider.ControllerClient go serviceAccountInformer.Informer().Run(wait.NeverStop) go nodeInfoController.Run(wait.NeverStop) + go flexCIDRController.Run(wait.NeverStop) // If the cluster is type OpenShift then the Tagging Controller // should be enabled. diff --git a/pkg/cloudprovider/providers/oci/flex_cidr_controller.go b/pkg/cloudprovider/providers/oci/flex_cidr_controller.go new file mode 100644 index 0000000000..982fede20c --- /dev/null +++ b/pkg/cloudprovider/providers/oci/flex_cidr_controller.go @@ -0,0 +1,203 @@ +package oci + +import ( + "context" + "fmt" + "time" + + "github.com/oracle/oci-cloud-controller-manager/pkg/flexcidr" + "github.com/oracle/oci-cloud-controller-manager/pkg/oci/client" + "github.com/oracle/oci-go-sdk/v65/core" + "github.com/pkg/errors" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" +) + +const flexCIDRRetryDelay = time.Minute + +type FlexCIDRController struct { + nodeInformer coreinformers.NodeInformer + serviceInformer coreinformers.ServiceInformer + kubeClient clientset.Interface + cloud *CloudProvider + queue workqueue.RateLimitingInterface + logger *zap.SugaredLogger + instanceCache cache.Store + ociClient client.Interface +} + +func NewFlexCIDRController( + nodeInformer coreinformers.NodeInformer, + serviceInformer coreinformers.ServiceInformer, + kubeClient clientset.Interface, + cloud *CloudProvider, + logger *zap.SugaredLogger, + instanceCache cache.Store, + ociClient client.Interface) *FlexCIDRController { + + controller := &FlexCIDRController{ + nodeInformer: nodeInformer, + serviceInformer: serviceInformer, + kubeClient: kubeClient, + cloud: cloud, + queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), + logger: logger, + instanceCache: instanceCache, + ociClient: ociClient, + } + + controller.nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + node := obj.(*v1.Node) + controller.queue.Add(node.Name) + }, + UpdateFunc: func(_, newObj interface{}) { + node := newObj.(*v1.Node) + controller.queue.Add(node.Name) + }, + }) + + return controller +} + +func (fcc *FlexCIDRController) Run(stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + defer fcc.queue.ShutDown() + + fcc.logger.Info("Starting flex CIDR controller") + + if !cache.WaitForCacheSync(stopCh, fcc.nodeInformer.Informer().HasSynced, fcc.serviceInformer.Informer().HasSynced) { + utilruntime.HandleError(fmt.Errorf("timed out waiting for flex CIDR controller caches to sync")) + return + } + + wait.Until(fcc.runWorker, time.Second, stopCh) +} + +func (fcc *FlexCIDRController) runWorker() { + for fcc.processNextItem() { + } +} + +func (fcc *FlexCIDRController) processNextItem() bool { + key, quit := fcc.queue.Get() + if quit { + return false + } + defer fcc.queue.Done(key) + + if err := fcc.processItem(key.(string)); err != nil { + fcc.logger.Errorf("Error processing flex CIDR for node %s (will retry): %v", key, err) + fcc.queue.AddRateLimited(key) + } else { + fcc.queue.Forget(key) + } + + return true +} + +func (fcc *FlexCIDRController) processItem(key string) error { + logger := fcc.logger.With("node", key) + + node, err := fcc.nodeInformer.Lister().Get(key) + if err != nil { + return err + } + + if len(node.Spec.PodCIDRs) > 0 && len(node.Spec.ProviderID) == 0 { + logger.Debug("node already has podCIDRs but providerID is empty, skipping") + return nil + } + + instance, instanceID, err := fcc.getInstanceByNode(node, logger) + if err != nil { + return err + } + if instance == nil { + return nil + } + + if err := fcc.instanceCache.Add(instance); err != nil { + logger.With(zap.Error(err)).Debug("failed to add instance to cache") + } + + if instance.LifecycleState != core.InstanceLifecycleStateRunning { + logger.Infof("instance %s not running yet, requeueing", instanceID) + fcc.queue.AddAfter(key, flexCIDRRetryDelay) + return nil + } + + config, hasConfig := flexcidr.ParsePrimaryVnicConfig(instance) + if !hasConfig { + logger.Debug("instance metadata does not include flex CIDR configuration, skipping") + return nil + } + + clusterIPFamily, err := flexcidr.GetClusterIpFamily(context.Background(), fcc.serviceInformer.Lister()) + if err != nil { + logger.With(zap.Error(err)).Info("cluster IP family not ready yet, requeueing") + fcc.queue.AddAfter(key, flexCIDRRetryDelay) + return nil + } + + primaryVNIC, err := fcc.ociClient.Compute().GetPrimaryVNICForInstance(context.Background(), *instance.CompartmentId, instanceID) + if err != nil { + return errors.Wrap(err, "GetPrimaryVNICForInstance") + } + + flexCIDRManager := &flexcidr.FlexCIDR{ + Logger: logger, + PrimaryVnicConfig: config, + ClusterIpFamily: clusterIPFamily, + OciCoreClient: fcc.ociClient.Networking(nil), + } + + flexCIDRs, err := flexCIDRManager.GetOrCreateFlexCidrList(*primaryVNIC.Id) + if err != nil { + return err + } + if !flexCIDRManager.ValidateFlexCidrList(flexCIDRs) { + return fmt.Errorf("computed flex CIDRs %v are invalid", flexCIDRs) + } + if flexcidr.StringSlicesEqualIgnoreOrder(node.Spec.PodCIDRs, flexCIDRs) { + logger.Debugf("node already has expected podCIDRs %v", flexCIDRs) + return nil + } + + return flexcidr.PatchNodePodCIDRs(context.Background(), fcc.kubeClient, node.Name, flexCIDRs, logger) +} + +func (fcc *FlexCIDRController) getInstanceByNode(node *v1.Node, logger *zap.SugaredLogger) (*core.Instance, string, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + providerID := node.Spec.ProviderID + var err error + if providerID == "" { + providerID, err = fcc.cloud.InstanceID(ctx, types.NodeName(node.Name)) + if err != nil { + return nil, "", err + } + } + + instanceID, err := MapProviderIDToResourceID(providerID) + if err != nil { + logger.With(zap.Error(err)).Error("failed to map providerID to instanceID") + return nil, "", err + } + + instance, err := fcc.ociClient.Compute().GetInstance(ctx, instanceID) + if err != nil { + logger.With(zap.Error(err)).Error("failed to fetch instance") + return nil, "", err + } + + return instance, instanceID, nil +} diff --git a/pkg/cloudprovider/providers/oci/instances_test.go b/pkg/cloudprovider/providers/oci/instances_test.go index 58f3015404..3214c1db05 100644 --- a/pkg/cloudprovider/providers/oci/instances_test.go +++ b/pkg/cloudprovider/providers/oci/instances_test.go @@ -1037,6 +1037,10 @@ func (c *MockVirtualNetworkClient) CreatePrivateIp(ctx context.Context, vnicID s return nil, nil } +func (c *MockVirtualNetworkClient) CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) { + return core.PrivateIp{}, nil +} + func (c *MockVirtualNetworkClient) GetIpv6(ctx context.Context, id string) (*core.Ipv6, error) { return &core.Ipv6{}, nil } @@ -1057,6 +1061,10 @@ func (c *MockVirtualNetworkClient) CreateIpv6(ctx context.Context, vnicID string return &core.Ipv6{}, nil } +func (c *MockVirtualNetworkClient) CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) { + return core.Ipv6{}, nil +} + func (c *MockVirtualNetworkClient) GetSubnet(ctx context.Context, id string) (*core.Subnet, error) { if subnet, ok := subnets[id]; ok { return subnet, nil @@ -1423,6 +1431,10 @@ func (MockIdentityClient) GetAvailabilityDomainByName(ctx context.Context, compa return nil, nil } +func (MockIdentityClient) ListAvailabilityDomains(ctx context.Context, compartmentID string) ([]identity.AvailabilityDomain, error) { + return nil, nil +} + type mockInstanceCache struct{} func (m mockInstanceCache) Add(obj interface{}) error { diff --git a/pkg/csi/driver/bv_controller_test.go b/pkg/csi/driver/bv_controller_test.go index 8657121410..2cb52411cb 100644 --- a/pkg/csi/driver/bv_controller_test.go +++ b/pkg/csi/driver/bv_controller_test.go @@ -487,10 +487,26 @@ func (c *MockVirtualNetworkClient) CreatePrivateIp(ctx context.Context, vnicId s return &core.PrivateIp{}, nil } +func (c *MockVirtualNetworkClient) CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) { + return core.PrivateIp{}, nil +} + func (c *MockVirtualNetworkClient) ListPrivateIps(ctx context.Context, id string) ([]core.PrivateIp, error) { return []core.PrivateIp{}, nil } +func (c *MockVirtualNetworkClient) ListIpv6s(ctx context.Context, vnicId string) ([]core.Ipv6, error) { + return []core.Ipv6{}, nil +} + +func (c *MockVirtualNetworkClient) CreateIpv6(ctx context.Context, vnicID string) (*core.Ipv6, error) { + return &core.Ipv6{}, nil +} + +func (c *MockVirtualNetworkClient) CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) { + return core.Ipv6{}, nil +} + func (c *MockVirtualNetworkClient) GetSubnet(ctx context.Context, id string) (*core.Subnet, error) { if strings.EqualFold(id, "ocid1.invalid-subnet") { return nil, errors.New("Internal Error.") diff --git a/pkg/flexcidr/flexcidr.go b/pkg/flexcidr/flexcidr.go new file mode 100644 index 0000000000..24e2641fab --- /dev/null +++ b/pkg/flexcidr/flexcidr.go @@ -0,0 +1,497 @@ +package flexcidr + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/bits" + "math/rand" + "net" + "net/http" + "sync" + "time" + + ociclient "github.com/oracle/oci-cloud-controller-manager/pkg/oci/client" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/core" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/util/retry" +) + +var retryRNG = rand.New(rand.NewSource(time.Now().UnixNano())) + +const ( + podCIDRTag = "pod-cidr" + primaryVnicMetadataKey = "flexcidr-primary-vnic" + createTimeout = 55 * time.Second + listTimeout = 25 * time.Second + defaultNamespace = "default" +) + +var initRetryOnce sync.Once + +func initOCIRetry() { + initRetryOnce.Do(func() { + p := common.DefaultRetryPolicy() + common.GlobalRetry = &p + }) +} + +func init() { initOCIRetry() } + +func runWithRateLimitRetry(ctx context.Context, logger *zap.SugaredLogger, operation string, fn func(context.Context) error) error { + policy := common.NewRetryPolicyWithOptions( + common.WithMaximumNumberAttempts(6), + common.WithShouldRetryOperation(func(r common.OCIOperationResponse) bool { + return isRateLimitError(r.Error) + }), + common.WithNextDuration(func(r common.OCIOperationResponse) time.Duration { + attempt := float64(r.AttemptNumber - 1) + base := math.Pow(2, attempt) + jitter := 1 + (retryRNG.Float64()-0.5)*0.2 + return time.Duration(base * jitter * float64(time.Second)) + }), + ) + + maxAttempts := policy.MaximumNumberAttempts + var lastErr error + + for attempt := uint(1); maxAttempts == 0 || attempt <= maxAttempts; attempt++ { + opErr := fn(ctx) + lastErr = opErr + operationResponse := common.OCIOperationResponse{Error: opErr, AttemptNumber: attempt} + + if !policy.ShouldRetryOperation(operationResponse) { + return opErr + } + + backoff := policy.NextDuration(operationResponse) + if logger != nil { + logger.Warnf("%s hit rate limit on attempt %d, retrying in %s", operation, attempt, backoff) + } + if deadline, ok := ctx.Deadline(); ok && time.Now().Add(backoff).After(deadline) { + return fmt.Errorf("%s retry exceeded context deadline: %w", operation, context.DeadlineExceeded) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(backoff): + } + } + + if lastErr == nil { + lastErr = fmt.Errorf("operation %s reached retry limit", operation) + } + return fmt.Errorf("%s retry exceeded maximum attempts: %w", operation, lastErr) +} + +func isRateLimitError(err error) bool { + if err == nil { + return false + } + var serviceErr common.ServiceError + if errors.As(err, &serviceErr) { + return serviceErr.GetHTTPStatusCode() == http.StatusTooManyRequests + } + return false +} + +type IpFamily struct { + IPv4 string + IPv6 string +} + +type PrimaryVnicConfig struct { + IPCount *int `json:"ip-count"` + CIDRBlocks []string `json:"cidr-blocks"` +} + +func ParsePrimaryVnicConfig(instance *core.Instance) (PrimaryVnicConfig, bool) { + for k, v := range instance.Metadata { + if k == primaryVnicMetadataKey { + var config PrimaryVnicConfig + if err := json.Unmarshal([]byte(v), &config); err != nil { + return PrimaryVnicConfig{}, false + } + return config, true + } + } + return PrimaryVnicConfig{}, false +} + +func GetClusterIpFamily(ctx context.Context, serviceLister corelisters.ServiceLister) (IpFamily, error) { + svc, err := serviceLister.Services(defaultNamespace).Get("kubernetes") + if err != nil { + return IpFamily{}, err + } + var family IpFamily + ipFamilies := svc.Spec.IPFamilies + if len(ipFamilies) == 0 || len(ipFamilies) > 2 { + return family, fmt.Errorf("IPFamily unset/invalid") + } + for _, ipFamily := range ipFamilies { + if ipFamily == corev1.IPv4Protocol { + family.IPv4 = "IPv4" + } + if ipFamily == corev1.IPv6Protocol { + family.IPv6 = "IPv6" + } + } + return family, nil +} + +func PatchNodePodCIDRs(ctx context.Context, kubeClient kubernetes.Interface, nodeName string, podCIDRs []string, logger *zap.SugaredLogger) error { + if len(podCIDRs) == 0 { + return fmt.Errorf("no PodCIDRs computed") + } + + node, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + nodeDry := node.DeepCopy() + nodeDry.Spec.PodCIDR = podCIDRs[0] + nodeDry.Spec.PodCIDRs = append([]string(nil), podCIDRs...) + if _, err := kubeClient.CoreV1().Nodes().Update(ctx, nodeDry, metav1.UpdateOptions{DryRun: []string{metav1.DryRunAll}}); err != nil { + if logger != nil { + logger.Errorf("dry-run update failed for node %s: %v", nodeName, err) + } + return err + } + + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + currentNode, getErr := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if getErr != nil { + return getErr + } + currentNode.Spec.PodCIDR = podCIDRs[0] + currentNode.Spec.PodCIDRs = append([]string(nil), podCIDRs...) + _, updateErr := kubeClient.CoreV1().Nodes().Update(ctx, currentNode, metav1.UpdateOptions{}) + return updateErr + }); err != nil { + if logger != nil { + logger.Errorf("failed to update node %s: %v", nodeName, err) + } + return err + } + + updatedNode, err := kubeClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + if updatedNode.Spec.PodCIDR != podCIDRs[0] { + return fmt.Errorf("post-update check: spec.podCIDR=%q (expected %q)", updatedNode.Spec.PodCIDR, podCIDRs[0]) + } + if !StringSlicesEqualIgnoreOrder(updatedNode.Spec.PodCIDRs, podCIDRs) { + return fmt.Errorf("post-update check: spec.podCIDRs=%v (expected %v)", updatedNode.Spec.PodCIDRs, podCIDRs) + } + + if logger != nil { + logger.Infof("successfully updated node %s podCIDRs to %v", nodeName, podCIDRs) + } + return nil +} + +func StringSlicesEqualIgnoreOrder(a, b []string) bool { + if len(a) != len(b) { + return false + } + m := make(map[string]int, len(a)) + for _, s := range a { + m[s]++ + } + for _, s := range b { + m[s]-- + if m[s] < 0 { + return false + } + } + for _, v := range m { + if v != 0 { + return false + } + } + return true +} + +func Ipv4PrefixFromCount(ipCount int) (int, error) { + if ipCount <= 0 || (ipCount&(ipCount-1)) != 0 { + return 0, fmt.Errorf("ipCount must be a power of 2, got %d", ipCount) + } + k := bits.Len(uint(ipCount)) - 1 + pfx := 32 - k + if pfx < 18 { + return 0, fmt.Errorf("ipCount=%d yields /%d; OCI requires cidrPrefixLength >= 18 (max ipCount 16384)", ipCount, pfx) + } + if pfx > 30 { + return 0, fmt.Errorf("ipCount=%d yields /%d; must be <= /30 (min ipCount 4)", ipCount, pfx) + } + return pfx, nil +} + +func Ipv6PrefixFromCount(ipCount int) (int, error) { + if ipCount <= 0 || (ipCount&(ipCount-1)) != 0 { + return 0, fmt.Errorf("ipCount must be a power of 2, got %d", ipCount) + } + k := bits.Len(uint(ipCount)) - 1 + pfx := 128 - k + validPfx := pfx / 4 * 4 + if pfx < 80 { + return 0, fmt.Errorf("ipCount=%d yields /%d; must be >= /80 (max ipCount 2^48)", ipCount, pfx) + } + return validPfx, nil +} + +func getCIDRsByFamily(cidrBlocks []string) ([]string, []string, error) { + var ipv4CidrBlocks, ipv6CidrBlocks []string + + for _, cidr := range cidrBlocks { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return nil, nil, fmt.Errorf("invalid CIDR block in %s.cidrBlocks: %q: %w", primaryVnicMetadataKey, cidr, err) + } + + if ip.To4() != nil { + ipv4CidrBlocks = append(ipv4CidrBlocks, cidr) + } else { + ipv6CidrBlocks = append(ipv6CidrBlocks, cidr) + } + } + + return ipv4CidrBlocks, ipv6CidrBlocks, nil +} + +func formatIpCidr(ip string, mask *int) string { + return fmt.Sprintf("%s/%d", ip, *mask) +} + +type FlexCIDR struct { + Logger *zap.SugaredLogger + PrimaryVnicConfig PrimaryVnicConfig + ClusterIpFamily IpFamily + OciCoreClient ociclient.NetworkingInterface +} + +func (f *FlexCIDR) validateCidrBlocks(ipv4Blocks []string, ipv6Blocks []string) (string, string, error) { + if f.ClusterIpFamily.IPv4 == "" && len(ipv4Blocks) > 0 { + return "", "", fmt.Errorf("IPv4 CIDR is not allowed for this cluster but provided: %v", ipv4Blocks) + } + if f.ClusterIpFamily.IPv6 == "" && len(ipv6Blocks) > 0 { + return "", "", fmt.Errorf("IPv6 CIDR is not allowed for this cluster but provided: %v", ipv6Blocks) + } + if len(ipv4Blocks) > 1 || len(ipv6Blocks) > 1 { + return "", "", fmt.Errorf("only one IPv4 CIDR block and one IPv6 CIDR block are allowed for DualStack cluster; found %d IPv4 and %d IPv6 CIDR blocks", len(ipv4Blocks), len(ipv6Blocks)) + } + var ipv4, ipv6 string + if len(ipv4Blocks) == 1 { + ipv4 = ipv4Blocks[0] + } + if len(ipv6Blocks) == 1 { + ipv6 = ipv6Blocks[0] + } + return ipv4, ipv6, nil +} + +func (f *FlexCIDR) getCidrBlocks() (string, string, error) { + if f.Logger != nil { + f.Logger.Infof("PrimaryVnicConfig CIDR blocks: %v", f.PrimaryVnicConfig.CIDRBlocks) + } + ipv4Blocks, ipv6Blocks, err := getCIDRsByFamily(f.PrimaryVnicConfig.CIDRBlocks) + if err != nil { + return "", "", err + } + return f.validateCidrBlocks(ipv4Blocks, ipv6Blocks) +} + +func (f *FlexCIDR) ValidateFlexCidrList(flexCidrs []string) bool { + if len(flexCidrs) == 0 { + if f.Logger != nil { + f.Logger.Errorf("flexCidrs is empty") + } + return false + } + for _, flexCidr := range flexCidrs { + if flexCidr == "" { + if f.Logger != nil { + f.Logger.Errorf("flexCidrs contains empty string") + } + return false + } + } + if f.ClusterIpFamily.IPv4 != "" && f.ClusterIpFamily.IPv6 != "" && len(flexCidrs) != 2 { + if f.Logger != nil { + f.Logger.Errorf("existing flexCidrs %v for dual stack should contain both IPv4 and IPv6 CIDRs", flexCidrs) + } + return false + } + if (f.ClusterIpFamily.IPv4 != "" && f.ClusterIpFamily.IPv6 == "") || (f.ClusterIpFamily.IPv4 == "" && f.ClusterIpFamily.IPv6 != "") { + if len(flexCidrs) != 1 { + if f.Logger != nil { + f.Logger.Errorf("flexCidrs %v is not valid for single stack cluster", flexCidrs) + } + return false + } + } + return true +} + +func (f *FlexCIDR) GetFlexCidrList(primaryVnicID string) ([]string, bool) { + ctx, cancel := context.WithTimeout(context.Background(), listTimeout) + defer cancel() + + var flexCidrs []string + + if f.ClusterIpFamily.IPv4 != "" { + privateIPs, err := f.OciCoreClient.ListPrivateIps(ctx, primaryVnicID) + if err != nil { + if f.Logger != nil { + f.Logger.Errorf("failed to list private IPs for VNIC %s: %v", primaryVnicID, err) + } + return flexCidrs, false + } + + for _, privateIP := range privateIPs { + if _, ok := privateIP.FreeformTags[podCIDRTag]; ok { + flexCidrs = append(flexCidrs, formatIpCidr(*privateIP.IpAddress, privateIP.CidrPrefixLength)) + } + } + } + + if f.ClusterIpFamily.IPv6 != "" { + ipv6s, err := f.OciCoreClient.ListIpv6s(ctx, primaryVnicID) + if err != nil { + if f.Logger != nil { + f.Logger.Errorf("failed to list IPv6s for VNIC %s: %v", primaryVnicID, err) + } + return flexCidrs, false + } + + for _, ipv6 := range ipv6s { + if _, ok := ipv6.FreeformTags[podCIDRTag]; ok { + flexCidrs = append(flexCidrs, formatIpCidr(*ipv6.IpAddress, ipv6.CidrPrefixLength)) + } + } + } + + return flexCidrs, len(flexCidrs) > 0 +} + +func (f *FlexCIDR) CreateFlexCidr(primaryVnicID string, isIPv4 bool, isIPv6 bool) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), createTimeout) + defer cancel() + + flexCidr := "" + + ipv4CidrBlock, ipv6CidrBlock, err := f.getCidrBlocks() + if err != nil { + return flexCidr, err + } + + if f.PrimaryVnicConfig.IPCount == nil { + return "", fmt.Errorf("primaryVNIC.ipCount is nil") + } + ipCount := *f.PrimaryVnicConfig.IPCount + + if isIPv4 { + cidrPrefixLength, err := Ipv4PrefixFromCount(ipCount) + if err != nil { + return "", err + } + createPrivateIPDetails := core.CreatePrivateIpDetails{ + VnicId: common.String(primaryVnicID), + CidrPrefixLength: common.Int(cidrPrefixLength), + FreeformTags: map[string]string{podCIDRTag: "true"}, + } + if ipv4CidrBlock != "" { + createPrivateIPDetails.Ipv4SubnetCidrAtCreation = common.String(ipv4CidrBlock) + } + privateIP, err := f.OciCoreClient.CreatePrivateIpWithRequest(ctx, core.CreatePrivateIpRequest{ + CreatePrivateIpDetails: createPrivateIPDetails, + }) + if err != nil { + return flexCidr, fmt.Errorf("failed to assign flex CIDR to IPv4 VNIC: %w", err) + } + ipv4Address := *privateIP.IpAddress + parsedIP := net.ParseIP(ipv4Address) + if parsedIP == nil || parsedIP.To4() == nil { + return flexCidr, fmt.Errorf("flex CIDR address (%s) returned by VCN is not a valid IPv4 address", ipv4Address) + } + flexCidr = formatIpCidr(ipv4Address, privateIP.CidrPrefixLength) + } + + if isIPv6 { + cidrPrefixLength, err := Ipv6PrefixFromCount(ipCount) + if err != nil { + return "", err + } + createIPv6Details := core.CreateIpv6Details{ + VnicId: common.String(primaryVnicID), + CidrPrefixLength: common.Int(cidrPrefixLength), + FreeformTags: map[string]string{podCIDRTag: "true"}, + } + if ipv6CidrBlock != "" { + createIPv6Details.Ipv6SubnetCidr = common.String(ipv6CidrBlock) + } + ipv6, err := f.OciCoreClient.CreateIpv6WithRequest(ctx, core.CreateIpv6Request{ + CreateIpv6Details: createIPv6Details, + }) + if err != nil { + return flexCidr, fmt.Errorf("failed to assign flex CIDR to IPv6 VNIC: %w", err) + } + ipv6Address := *ipv6.IpAddress + parsedIP := net.ParseIP(ipv6Address) + if parsedIP == nil || parsedIP.To4() != nil { + return flexCidr, fmt.Errorf("flex CIDR address (%s) returned by VCN is not a valid IPv6 address", ipv6Address) + } + flexCidr = formatIpCidr(ipv6Address, ipv6.CidrPrefixLength) + } + + return flexCidr, nil +} + +func (f *FlexCIDR) GetOrCreateFlexCidrList(primaryVnicID string) ([]string, error) { + var flexCidrs []string + + existingFlexCIDRs, ok := f.GetFlexCidrList(primaryVnicID) + if ok { + if f.Logger != nil { + f.Logger.Infof("flexCidrs %v already exist on primary VNIC %s", existingFlexCIDRs, primaryVnicID) + } + if f.ValidateFlexCidrList(existingFlexCIDRs) { + return existingFlexCIDRs, nil + } + return nil, fmt.Errorf("flexCidrs %v is invalid", existingFlexCIDRs) + } + + if f.ClusterIpFamily.IPv4 != "" { + ipv4FlexCIDR, err := f.CreateFlexCidr(primaryVnicID, true, false) + if err != nil { + return nil, err + } + flexCidrs = append(flexCidrs, ipv4FlexCIDR) + } + + if f.ClusterIpFamily.IPv6 != "" { + ipv6FlexCIDR, err := f.CreateFlexCidr(primaryVnicID, false, true) + if err != nil { + return nil, err + } + flexCidrs = append(flexCidrs, ipv6FlexCIDR) + } + + if len(flexCidrs) == 0 { + return nil, apierrors.NewBadRequest("no flex CIDRs created") + } + + return flexCidrs, nil +} diff --git a/pkg/flexcidr/flexcidr_test.go b/pkg/flexcidr/flexcidr_test.go new file mode 100644 index 0000000000..05ede90a46 --- /dev/null +++ b/pkg/flexcidr/flexcidr_test.go @@ -0,0 +1,430 @@ +package flexcidr + +import ( + "context" + "net/http" + "reflect" + "strings" + "testing" + + ociclient "github.com/oracle/oci-cloud-controller-manager/pkg/oci/client" + "github.com/oracle/oci-go-sdk/v65/common" + "github.com/oracle/oci-go-sdk/v65/core" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +type requireAssertions struct{} + +var require requireAssertions + +func (requireAssertions) True(t *testing.T, value bool, msgAndArgs ...interface{}) { + t.Helper() + if !value { + t.Fatal(msgAndArgs...) + } +} + +func (requireAssertions) False(t *testing.T, value bool, msgAndArgs ...interface{}) { + t.Helper() + if value { + t.Fatal(msgAndArgs...) + } +} + +func (requireAssertions) NoError(t *testing.T, err error, msgAndArgs ...interface{}) { + t.Helper() + if err != nil { + t.Fatal(append([]interface{}{err}, msgAndArgs...)...) + } +} + +func (requireAssertions) Error(t *testing.T, err error, msgAndArgs ...interface{}) { + t.Helper() + if err == nil { + t.Fatal(msgAndArgs...) + } +} + +func (requireAssertions) Equal(t *testing.T, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + t.Helper() + if !reflect.DeepEqual(expected, actual) { + t.Fatal(append([]interface{}{"expected", expected, "actual", actual}, msgAndArgs...)...) + } +} + +func (requireAssertions) ElementsMatch(t *testing.T, expected []string, actual []string, msgAndArgs ...interface{}) { + t.Helper() + if !StringSlicesEqualIgnoreOrder(expected, actual) { + t.Fatal(append([]interface{}{"expected", expected, "actual", actual}, msgAndArgs...)...) + } +} + +func (requireAssertions) NotNil(t *testing.T, value interface{}, msgAndArgs ...interface{}) { + t.Helper() + if value == nil { + t.Fatal(msgAndArgs...) + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + if rv.IsNil() { + t.Fatal(msgAndArgs...) + } + } +} + +func (requireAssertions) Nil(t *testing.T, value interface{}, msgAndArgs ...interface{}) { + t.Helper() + if value == nil { + return + } + rv := reflect.ValueOf(value) + switch rv.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + if rv.IsNil() { + return + } + } + if value != nil { + t.Fatal(msgAndArgs...) + } +} + +func (requireAssertions) Contains(t *testing.T, s string, contains string, msgAndArgs ...interface{}) { + t.Helper() + if !strings.Contains(s, contains) { + t.Fatal(append([]interface{}{"expected", s, "to contain", contains}, msgAndArgs...)...) + } +} + +type fakeServiceError struct { + status int +} + +func (e fakeServiceError) Error() string { return "service error" } +func (e fakeServiceError) GetCode() string { return "TooManyRequests" } +func (e fakeServiceError) GetMessage() string { return "rate limited" } +func (e fakeServiceError) GetHTTPStatusCode() int { return e.status } +func (e fakeServiceError) GetOpcRequestID() string { return "opc-request-id" } + +type testOciCoreClient struct { + ociclient.NetworkingInterface + + listPrivateIpsResp []core.PrivateIp + listPrivateIpsErr error + listIpv6sResp []core.Ipv6 + listIpv6sErr error + + createPrivateIpResp core.PrivateIp + createPrivateIpErr error + lastCreatePrivate *core.CreatePrivateIpRequest + + createIpv6Resp core.Ipv6 + createIpv6Err error + lastCreateIpv6 *core.CreateIpv6Request +} + +func (m *testOciCoreClient) ListPrivateIps(_ context.Context, _ string) ([]core.PrivateIp, error) { + return m.listPrivateIpsResp, m.listPrivateIpsErr +} + +func (m *testOciCoreClient) ListIpv6s(_ context.Context, _ string) ([]core.Ipv6, error) { + return m.listIpv6sResp, m.listIpv6sErr +} + +func (m *testOciCoreClient) CreatePrivateIpWithRequest(_ context.Context, req core.CreatePrivateIpRequest) (core.PrivateIp, error) { + m.lastCreatePrivate = &req + return m.createPrivateIpResp, m.createPrivateIpErr +} + +func (m *testOciCoreClient) CreateIpv6WithRequest(_ context.Context, req core.CreateIpv6Request) (core.Ipv6, error) { + m.lastCreateIpv6 = &req + return m.createIpv6Resp, m.createIpv6Err +} + +func testLogger() *zap.SugaredLogger { + return zap.NewNop().Sugar() +} + +func TestIsRateLimitError(t *testing.T) { + require.False(t, isRateLimitError(nil)) + require.False(t, isRateLimitError(context.DeadlineExceeded)) + require.False(t, isRateLimitError(fakeServiceError{status: http.StatusBadRequest})) + require.True(t, isRateLimitError(fakeServiceError{status: http.StatusTooManyRequests})) +} + +func TestParsePrimaryVnicConfig(t *testing.T) { + instance := &core.Instance{Metadata: map[string]string{primaryVnicMetadataKey: `{"ip-count":16,"cidr-blocks":["10.0.0.0/24"]}`}} + cfg, ok := ParsePrimaryVnicConfig(instance) + require.True(t, ok) + require.NotNil(t, cfg.IPCount) + require.Equal(t, 16, *cfg.IPCount) + require.Equal(t, []string{"10.0.0.0/24"}, cfg.CIDRBlocks) + + _, ok = ParsePrimaryVnicConfig(&core.Instance{Metadata: map[string]string{primaryVnicMetadataKey: `{"ip-count":`}}) + require.False(t, ok) + + _, ok = ParsePrimaryVnicConfig(&core.Instance{Metadata: map[string]string{"other": "x"}}) + require.False(t, ok) +} + +func TestGetClusterIpFamily(t *testing.T) { + kubeClient := fake.NewSimpleClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "kubernetes", Namespace: defaultNamespace}, + Spec: corev1.ServiceSpec{IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}}, + }) + factory := informers.NewSharedInformerFactory(kubeClient, 0) + serviceInformer := factory.Core().V1().Services() + require.NoError(t, serviceInformer.Informer().GetStore().Add(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "kubernetes", Namespace: defaultNamespace}, + Spec: corev1.ServiceSpec{IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}}, + })) + + family, err := GetClusterIpFamily(context.Background(), serviceInformer.Lister()) + require.NoError(t, err) + require.Equal(t, IpFamily{IPv4: "IPv4", IPv6: "IPv6"}, family) +} + +func TestPatchNodePodCIDRs(t *testing.T) { + kubeClient := fake.NewSimpleClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "node-1"}, + }) + + err := PatchNodePodCIDRs(context.Background(), kubeClient, "node-1", []string{"10.0.0.0/24", "2001:db8::/80"}, testLogger()) + require.NoError(t, err) + + updated, err := kubeClient.CoreV1().Nodes().Get(context.Background(), "node-1", metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, "10.0.0.0/24", updated.Spec.PodCIDR) + require.ElementsMatch(t, []string{"10.0.0.0/24", "2001:db8::/80"}, updated.Spec.PodCIDRs) +} + +func TestStringSlicesEqualIgnoreOrder(t *testing.T) { + require.True(t, StringSlicesEqualIgnoreOrder([]string{"a", "b"}, []string{"b", "a"})) + require.True(t, StringSlicesEqualIgnoreOrder([]string{"a", "a", "b"}, []string{"a", "b", "a"})) + require.False(t, StringSlicesEqualIgnoreOrder([]string{"a"}, []string{"a", "b"})) + require.False(t, StringSlicesEqualIgnoreOrder([]string{"a", "b"}, []string{"a", "c"})) +} + +func TestIpv4PrefixFromCount(t *testing.T) { + tests := []struct { + name string + count int + prefix int + errText string + }{ + {name: "valid minimum", count: 4, prefix: 30}, + {name: "valid maximum", count: 16384, prefix: 18}, + {name: "not power of two", count: 3, errText: "power of 2"}, + {name: "too small prefix", count: 32768, errText: "requires cidrPrefixLength >= 18"}, + {name: "too large prefix", count: 2, errText: "must be <= /30"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, err := Ipv4PrefixFromCount(tt.count) + if tt.errText != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errText) + return + } + require.NoError(t, err) + require.Equal(t, tt.prefix, prefix) + }) + } +} + +func TestIpv6PrefixFromCount(t *testing.T) { + tests := []struct { + name string + count int + prefix int + errText string + }{ + {name: "valid rounds to nibble", count: 1024, prefix: 116}, + {name: "valid exact nibble", count: 65536, prefix: 112}, + {name: "not power of two", count: 7, errText: "power of 2"}, + {name: "too small prefix", count: 1 << 49, errText: "must be >= /80"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefix, err := Ipv6PrefixFromCount(tt.count) + if tt.errText != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.errText) + return + } + require.NoError(t, err) + require.Equal(t, tt.prefix, prefix) + }) + } +} + +func TestGetCIDRsByFamily(t *testing.T) { + ipv4, ipv6, err := getCIDRsByFamily([]string{"10.0.0.0/24", "2001:db8::/64"}) + require.NoError(t, err) + require.Equal(t, []string{"10.0.0.0/24"}, ipv4) + require.Equal(t, []string{"2001:db8::/64"}, ipv6) + + _, _, err = getCIDRsByFamily([]string{"invalid"}) + require.Error(t, err) +} + +func TestValidateCidrBlocks(t *testing.T) { + f := &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}} + ipv4, ipv6, err := f.validateCidrBlocks([]string{"10.0.0.0/24"}, []string{"2001:db8::/64"}) + require.NoError(t, err) + require.Equal(t, "10.0.0.0/24", ipv4) + require.Equal(t, "2001:db8::/64", ipv6) + + f = &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4"}} + _, _, err = f.validateCidrBlocks(nil, []string{"2001:db8::/64"}) + require.Error(t, err) + + f = &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}} + _, _, err = f.validateCidrBlocks([]string{"10.0.0.0/24", "10.1.0.0/24"}, nil) + require.Error(t, err) +} + +func TestValidateFlexCidrList(t *testing.T) { + fDual := &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}} + require.True(t, fDual.ValidateFlexCidrList([]string{"10.0.0.1/30", "2001:db8::1/116"})) + require.False(t, fDual.ValidateFlexCidrList([]string{"10.0.0.1/30"})) + + fSingle := &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4"}} + require.True(t, fSingle.ValidateFlexCidrList([]string{"10.0.0.1/30"})) + require.False(t, fSingle.ValidateFlexCidrList([]string{"10.0.0.1/30", "10.0.0.2/30"})) + require.False(t, fSingle.ValidateFlexCidrList([]string{""})) +} + +func TestGetFlexCidrList(t *testing.T) { + pfx4 := 30 + pfx6 := 116 + fakeClient := &testOciCoreClient{ + listPrivateIpsResp: []core.PrivateIp{ + {IpAddress: common.String("10.0.0.10"), CidrPrefixLength: &pfx4, FreeformTags: map[string]string{podCIDRTag: "true"}}, + {IpAddress: common.String("10.0.0.11"), CidrPrefixLength: &pfx4, FreeformTags: map[string]string{"other": "x"}}, + }, + listIpv6sResp: []core.Ipv6{ + {IpAddress: common.String("2001:db8::10"), CidrPrefixLength: &pfx6, FreeformTags: map[string]string{podCIDRTag: "true"}}, + }, + } + f := &FlexCIDR{Logger: testLogger(), ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}, OciCoreClient: fakeClient} + + cidrs, ok := f.GetFlexCidrList("vnic") + require.True(t, ok) + require.ElementsMatch(t, []string{"10.0.0.10/30", "2001:db8::10/116"}, cidrs) +} + +func TestCreateFlexCidrIPv4ConfiguresSubnetCidrWhenSet(t *testing.T) { + pfx := 22 + fakeClient := &testOciCoreClient{createPrivateIpResp: core.PrivateIp{IpAddress: common.String("10.0.1.5"), CidrPrefixLength: &pfx}} + ipCount := 1024 + f := &FlexCIDR{ + Logger: testLogger(), + PrimaryVnicConfig: PrimaryVnicConfig{IPCount: &ipCount, CIDRBlocks: []string{"10.0.0.0/24"}}, + ClusterIpFamily: IpFamily{IPv4: "IPv4"}, + OciCoreClient: fakeClient, + } + + cidr, err := f.CreateFlexCidr("vnic", true, false) + require.NoError(t, err) + require.Equal(t, "10.0.1.5/22", cidr) + require.NotNil(t, fakeClient.lastCreatePrivate) + require.NotNil(t, fakeClient.lastCreatePrivate.CreatePrivateIpDetails.Ipv4SubnetCidrAtCreation) + require.Equal(t, "10.0.0.0/24", *fakeClient.lastCreatePrivate.CreatePrivateIpDetails.Ipv4SubnetCidrAtCreation) +} + +func TestCreateFlexCidrIPv6ConfiguresSubnetCidrWhenSet(t *testing.T) { + pfx := 116 + fakeClient := &testOciCoreClient{createIpv6Resp: core.Ipv6{IpAddress: common.String("2001:db8::5"), CidrPrefixLength: &pfx}} + ipCount := 1024 + f := &FlexCIDR{ + Logger: testLogger(), + PrimaryVnicConfig: PrimaryVnicConfig{IPCount: &ipCount, CIDRBlocks: []string{"2001:db8:1::/64"}}, + ClusterIpFamily: IpFamily{IPv6: "IPv6"}, + OciCoreClient: fakeClient, + } + + cidr, err := f.CreateFlexCidr("vnic", false, true) + require.NoError(t, err) + require.Equal(t, "2001:db8::5/116", cidr) + require.NotNil(t, fakeClient.lastCreateIpv6) + require.NotNil(t, fakeClient.lastCreateIpv6.CreateIpv6Details.Ipv6SubnetCidr) + require.Equal(t, "2001:db8:1::/64", *fakeClient.lastCreateIpv6.CreateIpv6Details.Ipv6SubnetCidr) +} + +func TestCreateFlexCidrIPv6DoesNotConfigureSubnetCidrWhenUnset(t *testing.T) { + pfx := 116 + fakeClient := &testOciCoreClient{createIpv6Resp: core.Ipv6{IpAddress: common.String("2001:db8::6"), CidrPrefixLength: &pfx}} + ipCount := 1024 + f := &FlexCIDR{ + Logger: testLogger(), + PrimaryVnicConfig: PrimaryVnicConfig{IPCount: &ipCount}, + ClusterIpFamily: IpFamily{IPv6: "IPv6"}, + OciCoreClient: fakeClient, + } + + cidr, err := f.CreateFlexCidr("vnic", false, true) + require.NoError(t, err) + require.Equal(t, "2001:db8::6/116", cidr) + require.NotNil(t, fakeClient.lastCreateIpv6) + require.Nil(t, fakeClient.lastCreateIpv6.CreateIpv6Details.Ipv6SubnetCidr) +} + +func TestGetOrCreateFlexCidrList(t *testing.T) { + ipCount := 1024 + pfx4 := 22 + pfx6 := 116 + fakeClient := &testOciCoreClient{ + createPrivateIpResp: core.PrivateIp{IpAddress: common.String("10.0.1.5"), CidrPrefixLength: &pfx4}, + createIpv6Resp: core.Ipv6{IpAddress: common.String("2001:db8::7"), CidrPrefixLength: &pfx6}, + } + f := &FlexCIDR{ + Logger: testLogger(), + PrimaryVnicConfig: PrimaryVnicConfig{IPCount: &ipCount}, + ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}, + OciCoreClient: fakeClient, + } + + cidrs, err := f.GetOrCreateFlexCidrList("vnic") + require.NoError(t, err) + require.ElementsMatch(t, []string{"10.0.1.5/22", "2001:db8::7/116"}, cidrs) +} + +func TestFormatIpCidr(t *testing.T) { + pfx := 24 + require.Equal(t, "10.0.0.1/24", formatIpCidr("10.0.0.1", &pfx)) +} + +func TestGetOrCreateFlexCidrListRejectsInvalidExistingCIDRs(t *testing.T) { + pfx4 := 30 + fakeClient := &testOciCoreClient{ + listPrivateIpsResp: []core.PrivateIp{ + {IpAddress: common.String("10.0.0.10"), CidrPrefixLength: &pfx4, FreeformTags: map[string]string{podCIDRTag: "true"}}, + }, + } + f := &FlexCIDR{ + Logger: testLogger(), + ClusterIpFamily: IpFamily{IPv4: "IPv4", IPv6: "IPv6"}, + OciCoreClient: fakeClient, + } + + _, err := f.GetOrCreateFlexCidrList("vnic") + require.Error(t, err) + require.True(t, strings.Contains(err.Error(), "invalid")) +} + +func TestParsePrimaryVnicConfigCIDRBlocksOrderIndependent(t *testing.T) { + instance := &core.Instance{Metadata: map[string]string{primaryVnicMetadataKey: `{"ip-count":16,"cidr-blocks":["2001:db8::/64","10.0.0.0/24"]}`}} + cfg, ok := ParsePrimaryVnicConfig(instance) + require.True(t, ok) + require.NotNil(t, cfg.IPCount) + require.Equal(t, 16, *cfg.IPCount) + require.True(t, reflect.DeepEqual([]string{"2001:db8::/64", "10.0.0.0/24"}, cfg.CIDRBlocks)) +} diff --git a/pkg/oci/client/client.go b/pkg/oci/client/client.go index cb44b1ec67..ecf0805111 100644 --- a/pkg/oci/client/client.go +++ b/pkg/oci/client/client.go @@ -88,6 +88,8 @@ type virtualNetworkClient interface { GetPrivateIp(ctx context.Context, request core.GetPrivateIpRequest) (response core.GetPrivateIpResponse, err error) ListPrivateIps(ctx context.Context, request core.ListPrivateIpsRequest) (response core.ListPrivateIpsResponse, err error) CreatePrivateIp(ctx context.Context, request core.CreatePrivateIpRequest) (response core.CreatePrivateIpResponse, err error) + ListIpv6s(ctx context.Context, request core.ListIpv6sRequest) (response core.ListIpv6sResponse, err error) + CreateIpv6(ctx context.Context, request core.CreateIpv6Request) (response core.CreateIpv6Response, err error) GetPublicIpByIpAddress(ctx context.Context, request core.GetPublicIpByIpAddressRequest) (response core.GetPublicIpByIpAddressResponse, err error) GetIpv6(ctx context.Context, request core.GetIpv6Request) (response core.GetIpv6Response, err error) diff --git a/pkg/oci/client/client_test.go b/pkg/oci/client/client_test.go index 47fe1295e0..b741085216 100644 --- a/pkg/oci/client/client_test.go +++ b/pkg/oci/client/client_test.go @@ -239,6 +239,14 @@ func (c *mockVirtualNetworkClient) CreatePrivateIp(ctx context.Context, request return core.CreatePrivateIpResponse{}, nil } +func (c *mockVirtualNetworkClient) ListIpv6s(ctx context.Context, request core.ListIpv6sRequest) (response core.ListIpv6sResponse, err error) { + return core.ListIpv6sResponse{}, nil +} + +func (c *mockVirtualNetworkClient) CreateIpv6(ctx context.Context, request core.CreateIpv6Request) (response core.CreateIpv6Response, err error) { + return core.CreateIpv6Response{}, nil +} + func (c *mockVirtualNetworkClient) GetIpv6(ctx context.Context, request core.GetIpv6Request) (response core.GetIpv6Response, err error) { return core.GetIpv6Response{}, nil } diff --git a/pkg/oci/client/networking.go b/pkg/oci/client/networking.go index 722914fd74..672667cb63 100644 --- a/pkg/oci/client/networking.go +++ b/pkg/oci/client/networking.go @@ -41,7 +41,11 @@ type NetworkingInterface interface { ListPrivateIps(ctx context.Context, vnicId string) ([]core.PrivateIp, error) GetPrivateIp(ctx context.Context, id string) (*core.PrivateIp, error) CreatePrivateIp(ctx context.Context, vnicID string) (*core.PrivateIp, error) + CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) GetIpv6(ctx context.Context, id string) (*core.Ipv6, error) + ListIpv6s(ctx context.Context, vnicId string) ([]core.Ipv6, error) + CreateIpv6(ctx context.Context, vnicID string) (*core.Ipv6, error) + CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) GetPublicIpByIpAddress(ctx context.Context, id string) (*core.PublicIp, error) @@ -301,23 +305,31 @@ func (c *client) ListPrivateIps(ctx context.Context, vnicId string) ([]core.Priv } func (c *client) CreatePrivateIp(ctx context.Context, vnicId string) (*core.PrivateIp, error) { - if !c.rateLimiter.Writer.TryAccept() { - return nil, RateLimitError(false, "CreatePrivateIp") - } - requestMetadata := getDefaultRequestMetadata(c.requestMetadata) - resp, err := c.network.CreatePrivateIp(ctx, core.CreatePrivateIpRequest{ + privateIP, err := c.CreatePrivateIpWithRequest(ctx, core.CreatePrivateIpRequest{ CreatePrivateIpDetails: core.CreatePrivateIpDetails{ VnicId: &vnicId, }, - RequestMetadata: requestMetadata, }) + if err != nil { + return nil, err + } + return &privateIP, nil +} + +func (c *client) CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) { + if !c.rateLimiter.Writer.TryAccept() { + return core.PrivateIp{}, RateLimitError(false, "CreatePrivateIp") + } + requestMetadata := getDefaultRequestMetadata(c.requestMetadata) + request.RequestMetadata = requestMetadata + resp, err := c.network.CreatePrivateIp(ctx, request) incRequestCounter(err, createVerb, privateIPResource) if err != nil { - c.logger.With(vnicId).Infof("CreatePrivateIp failed %s", pointer.StringDeref(resp.OpcRequestId, "")) - return nil, errors.WithStack(err) + c.logger.Infof("CreatePrivateIp failed %s", pointer.StringDeref(resp.OpcRequestId, "")) + return core.PrivateIp{}, errors.WithStack(err) } - return &resp.PrivateIp, nil + return resp.PrivateIp, nil } func (c *client) GetIpv6(ctx context.Context, id string) (*core.Ipv6, error) { @@ -343,6 +355,60 @@ func (c *client) GetIpv6(ctx context.Context, id string) (*core.Ipv6, error) { return &resp.Ipv6, nil } +func (c *client) ListIpv6s(ctx context.Context, vnicId string) ([]core.Ipv6, error) { + var page *string + ipv6s := []core.Ipv6{} + for { + if !c.rateLimiter.Reader.TryAccept() { + return nil, RateLimitError(false, "ListIpv6s") + } + resp, err := c.network.ListIpv6s(ctx, core.ListIpv6sRequest{ + VnicId: &vnicId, + Page: page, + RequestMetadata: c.requestMetadata, + }) + incRequestCounter(err, listVerb, ipv6IPResource) + if err != nil { + c.logger.With(vnicId).Infof("ListIpv6s failed %s", pointer.StringDeref(resp.OpcRequestId, "")) + return nil, errors.WithStack(err) + } + ipv6s = append(ipv6s, resp.Items...) + if page = resp.OpcNextPage; page == nil { + break + } + } + + return ipv6s, nil +} + +func (c *client) CreateIpv6(ctx context.Context, vnicId string) (*core.Ipv6, error) { + ipv6, err := c.CreateIpv6WithRequest(ctx, core.CreateIpv6Request{ + CreateIpv6Details: core.CreateIpv6Details{ + VnicId: &vnicId, + }, + }) + if err != nil { + return nil, err + } + return &ipv6, nil +} + +func (c *client) CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) { + if !c.rateLimiter.Writer.TryAccept() { + return core.Ipv6{}, RateLimitError(false, "CreateIpv6") + } + requestMetadata := getDefaultRequestMetadata(c.requestMetadata) + request.RequestMetadata = requestMetadata + resp, err := c.network.CreateIpv6(ctx, request) + incRequestCounter(err, createVerb, ipv6IPResource) + if err != nil { + c.logger.Infof("CreateIpv6 failed %s", pointer.StringDeref(resp.OpcRequestId, "")) + return core.Ipv6{}, errors.WithStack(err) + } + + return resp.Ipv6, nil +} + func (c *client) CreateNetworkSecurityGroup(ctx context.Context, compartmentId, vcnId, displayName, serviceUid string) (*core.NetworkSecurityGroup, error) { if !c.rateLimiter.Writer.TryAccept() { return nil, RateLimitError(false, "CreateNetworkSecurityGroup") diff --git a/pkg/volume/provisioner/block/block_test.go b/pkg/volume/provisioner/block/block_test.go index 1c9882350b..57f62ca1bd 100644 --- a/pkg/volume/provisioner/block/block_test.go +++ b/pkg/volume/provisioner/block/block_test.go @@ -344,9 +344,26 @@ func (c *MockVirtualNetworkClient) ListPrivateIps(ctx context.Context, id string func (c *MockVirtualNetworkClient) CreatePrivateIp(ctx context.Context, vnicId string) (*core.PrivateIp, error) { return &core.PrivateIp{}, nil } + +func (c *MockVirtualNetworkClient) CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) { + return core.PrivateIp{}, nil +} + func (c *MockVirtualNetworkClient) GetIpv6(ctx context.Context, id string) (*core.Ipv6, error) { return &core.Ipv6{}, nil } + +func (c *MockVirtualNetworkClient) ListIpv6s(ctx context.Context, vnicId string) ([]core.Ipv6, error) { + return []core.Ipv6{}, nil +} + +func (c *MockVirtualNetworkClient) CreateIpv6(ctx context.Context, vnicID string) (*core.Ipv6, error) { + return &core.Ipv6{}, nil +} + +func (c *MockVirtualNetworkClient) CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) { + return core.Ipv6{}, nil +} func (c *MockVirtualNetworkClient) GetSubnet(ctx context.Context, id string) (*core.Subnet, error) { return nil, nil } diff --git a/pkg/volume/provisioner/fss/fss_test.go b/pkg/volume/provisioner/fss/fss_test.go index c3b464fbc6..5244fbd310 100644 --- a/pkg/volume/provisioner/fss/fss_test.go +++ b/pkg/volume/provisioner/fss/fss_test.go @@ -345,6 +345,10 @@ func (c *MockVirtualNetworkClient) CreatePrivateIp(ctx context.Context, vnicId s return &core.PrivateIp{}, nil } +func (c *MockVirtualNetworkClient) CreatePrivateIpWithRequest(ctx context.Context, request core.CreatePrivateIpRequest) (core.PrivateIp, error) { + return core.PrivateIp{}, nil +} + func (c *MockVirtualNetworkClient) ListIpv6s(ctx context.Context, vnicId string) ([]core.Ipv6, error) { return []core.Ipv6{}, nil } @@ -353,6 +357,10 @@ func (c *MockVirtualNetworkClient) CreateIpv6(ctx context.Context, vnicID string return &core.Ipv6{}, nil } +func (c *MockVirtualNetworkClient) CreateIpv6WithRequest(ctx context.Context, request core.CreateIpv6Request) (core.Ipv6, error) { + return core.Ipv6{}, nil +} + func (c *MockVirtualNetworkClient) GetSubnet(ctx context.Context, id string) (*core.Subnet, error) { return nil, nil }