improve k8s informer cache

pkg/config/config.go

@@ -316,6 +316,10 @@ func DisableKubeClientKeepAlive() bool {
 	return viper.GetBool(setting.ENVDisableKubeClientKeepAlive)
 }
 
+func KubeInformerIdleTimeoutMinutes() int {
+	return viper.GetInt(setting.ENVKubeInformerIdleTimeout)
+}
+
 func IsDocumentDB() bool {
 	return viper.GetBool(setting.ENVIsDocumentDB)
 }

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_deploy.go

@@ -158,12 +158,14 @@ func (c *DeployJobCtl) run(ctx context.Context) error {
 		return errors.New(msg)
 	}
 
-	c.informer, err = clientmanager.NewKubeClientManager().GetInformer(c.jobTaskSpec.ClusterID, c.namespace)
+	var releaseInformer func()
+	c.informer, releaseInformer, err = clientmanager.NewKubeClientManager().AcquireInformer(c.jobTaskSpec.ClusterID, c.namespace)
 	if err != nil {
 		msg := fmt.Sprintf("can't init k8s informer: %v", err)
 		logError(c.job, msg, c.logger)
 		return errors.New(msg)
 	}
+	defer releaseInformer()
 
 	c.istioClient, err = clientmanager.NewKubeClientManager().GetIstioClientSet(c.jobTaskSpec.ClusterID)
 	if err != nil {

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_freestyle.go

@@ -73,6 +73,7 @@ type FreestyleJobCtl struct {
 	kubeclient  crClient.Client
 	informer    informers.SharedInformerFactory
 	apiServer   crClient.Reader
+	releaseInf  func()
 	paths       *string
 	jobTaskSpec *commonmodels.JobTaskFreestyleSpec
 	ack         func()
@@ -130,6 +131,9 @@ func (c *FreestyleJobCtl) Run(ctx context.Context) {
 		if err := c.run(ctx); err != nil {
 			return
 		}
+		if c.releaseInf != nil {
+			defer c.releaseInf()
+		}
 		c.wait(ctx)
 		c.complete(ctx)
 	}
@@ -299,11 +303,12 @@ func (c *FreestyleJobCtl) run(ctx context.Context) error {
 	}
 
 	// set informer when job and cm have been created
-	informer, err := clientmanager.NewKubeClientManager().GetInformer(c.jobTaskSpec.Properties.ClusterID, c.jobTaskSpec.Properties.Namespace)
+	informer, releaseInformer, err := clientmanager.NewKubeClientManager().AcquireInformer(c.jobTaskSpec.Properties.ClusterID, c.jobTaskSpec.Properties.Namespace)
 	if err != nil {
 		return errors.Wrap(err, "get informer")
 	}
 	c.informer = informer
+	c.releaseInf = releaseInformer
 	c.logger.Infof("succeed to create job %s", c.job.K8sJobName)
 	return nil
 }

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_mse_gray_offline.go

@@ -97,12 +97,14 @@ func (c *MseGrayOfflineJobCtl) Run(ctx context.Context) {
 		logError(c.job, msg, c.logger)
 		return
 	}
-	c.informer, err = clientmanager.NewKubeClientManager().GetInformer(clusterID, c.jobTaskSpec.Namespace)
+	var releaseInformer func()
+	c.informer, releaseInformer, err = clientmanager.NewKubeClientManager().AcquireInformer(clusterID, c.jobTaskSpec.Namespace)
 	if err != nil {
 		msg := fmt.Sprintf("can't init k8s informer: %v", err)
 		logError(c.job, msg, c.logger)
 		return
 	}
+	defer releaseInformer()
 	selector := labels.Set{
 		types.ZadigReleaseTypeLabelKey:    types.ZadigReleaseTypeMseGray,
 		types.ZadigReleaseVersionLabelKey: c.jobTaskSpec.GrayTag,

pkg/microservice/aslan/core/common/service/workflowcontroller/jobcontroller/job_mse_gray_release.go

@@ -101,12 +101,14 @@ func (c *MseGrayReleaseJobCtl) Run(ctx context.Context) {
 		return
 	}
 
-	c.informer, err = clientmanager.NewKubeClientManager().GetInformer(clusterID, c.namespace)
+	var releaseInformer func()
+	c.informer, releaseInformer, err = clientmanager.NewKubeClientManager().AcquireInformer(clusterID, c.namespace)
 	if err != nil {
 		msg := fmt.Sprintf("can't init k8s informer: %v", err)
 		logError(c.job, msg, c.logger)
 		return
 	}
+	defer releaseInformer()
 
 	resources := make([]*unstructured.Unstructured, 0)
 	service := c.jobTaskSpec.GrayService

pkg/setting/consts.go

@@ -48,6 +48,7 @@ const (
 	ENVPodIP                      = "BE_POD_IP"
 	ENVNamespace                  = "BE_POD_NAMESPACE"
 	ENVDisableKubeClientKeepAlive = "DISABLE_KUBE_CLIENT_KEEP_ALIVE"
+	ENVKubeInformerIdleTimeout    = "KUBE_INFORMER_IDLE_TIMEOUT_MINUTES"
 
 	// Aslan
 	ENVLogLevel                  = "LOG_LEVEL"