From 55fb76b0d03234c6a46827a05afbc3f02d9a3134 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 19:10:03 -0300 Subject: [PATCH 01/11] feat(cfworkers): generate presigned state download URL, pass STATE_DOWNLOAD_URL to builder job --- .../deployment-operator-controller-manager.yaml | 4 ++++ internal/build/cfworkers.go | 3 +++ internal/build/s3presign.go | 14 ++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index 72eeadf..7aedac4 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -104,6 +104,10 @@ spec: - name: S3_LOGS_BUCKET value: {{ .logsBucket | quote }} {{- end }} + {{- if .stateBucket }} + - name: S3_STATE_BUCKET + value: {{ .stateBucket | quote }} + {{- end }} {{- end }} {{- end }} livenessProbe: diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index db89b94..4814f94 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -26,6 +26,7 @@ type presignedURLs struct { LogsUpload string CacheDownload string CacheUpload string + StateDownload string } // cfWorkersJobOpts are the inputs for NewJob. @@ -79,6 +80,7 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { {Name: "LOGS_UPLOAD_URL", Value: opts.presignedURLs.LogsUpload}, {Name: "CACHE_DOWNLOAD_URL", Value: opts.presignedURLs.CacheDownload}, {Name: "CACHE_UPLOAD_URL", Value: opts.presignedURLs.CacheUpload}, + {Name: "STATE_DOWNLOAD_URL", Value: opts.presignedURLs.StateDownload}, } if src.BranchRef != "" { env = append(env, corev1.EnvVar{Name: "BRANCH_REF", Value: src.BranchRef}) @@ -177,6 +179,7 @@ func CfWorkersConfigFromEnv() CfWorkersConfig { SecretAccessKey: os.Getenv("S3_SECRET_ACCESS_KEY"), LogsBucket: os.Getenv("S3_LOGS_BUCKET"), ArtifactsBucket: os.Getenv("S3_ARTIFACTS_BUCKET"), + StateBucket: os.Getenv("S3_STATE_BUCKET"), }, } } diff --git a/internal/build/s3presign.go b/internal/build/s3presign.go index 4d83c8d..993d06a 100644 --- a/internal/build/s3presign.go +++ b/internal/build/s3presign.go @@ -20,6 +20,7 @@ type S3Config struct { SecretAccessKey string LogsBucket string // bucket for build logs ArtifactsBucket string // bucket for build artifacts + StateBucket string // bucket for per-site state (defaults to ArtifactsBucket) } // GeneratepresignedURLs generates all presigned URLs the build job needs. @@ -63,9 +64,22 @@ func generatePresignedURLs(ctx context.Context, cfg S3Config, site, jobName stri return presignedURLs{}, fmt.Errorf("presigning cache upload: %w", err) } + stateBucket := cfg.StateBucket + if stateBucket == "" { + stateBucket = cfg.ArtifactsBucket + } + stateDownload, err := presigner.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(stateBucket), + Key: aws.String(fmt.Sprintf("cfworkers/%s/state.json", site)), + }, s3.WithPresignExpires(presignExpiry)) + if err != nil { + return presignedURLs{}, fmt.Errorf("presigning state download: %w", err) + } + return presignedURLs{ LogsUpload: logsUpload.URL, CacheDownload: cacheDownload.URL, CacheUpload: cacheUpload.URL, + StateDownload: stateDownload.URL, }, nil } From 730c6c82f279565941ed0cd0d2409ff3e22476b0 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 19:15:22 -0300 Subject: [PATCH 02/11] refactor(cfworkers): use site-state/ prefix and deco-admin-states bucket default --- internal/build/s3presign.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/build/s3presign.go b/internal/build/s3presign.go index 993d06a..5c1c907 100644 --- a/internal/build/s3presign.go +++ b/internal/build/s3presign.go @@ -66,11 +66,11 @@ func generatePresignedURLs(ctx context.Context, cfg S3Config, site, jobName stri stateBucket := cfg.StateBucket if stateBucket == "" { - stateBucket = cfg.ArtifactsBucket + stateBucket = "deco-admin-states" } stateDownload, err := presigner.PresignGetObject(ctx, &s3.GetObjectInput{ Bucket: aws.String(stateBucket), - Key: aws.String(fmt.Sprintf("cfworkers/%s/state.json", site)), + Key: aws.String(fmt.Sprintf("site-state/%s/state.json", site)), }, s3.WithPresignExpires(presignExpiry)) if err != nil { return presignedURLs{}, fmt.Errorf("presigning state download: %w", err) From 5a7c4b0fd3a1fc7e0152938b3eea759d28a61e4e Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 19:24:35 -0300 Subject: [PATCH 03/11] refactor(cfworkers): pass S3 credentials directly to builder instead of presigned URLs Cache and state are now accessed by the builder using S3_ARTIFACTS_BUCKET, S3_STATE_BUCKET, S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY env vars. --- internal/build/cfworkers.go | 15 ++++++------- internal/build/s3presign.go | 42 ++++--------------------------------- 2 files changed, 12 insertions(+), 45 deletions(-) diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index 4814f94..c061a7e 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -23,10 +23,7 @@ func JobName(commitSha, site string) string { // presignedURLs are the S3 presigned URLs the build job needs. type presignedURLs struct { - LogsUpload string - CacheDownload string - CacheUpload string - StateDownload string + LogsUpload string } // cfWorkersJobOpts are the inputs for NewJob. @@ -37,6 +34,7 @@ type cfWorkersJobOpts struct { CfApiToken string CfAccountId string presignedURLs presignedURLs + S3 S3Config // SourceOverride replaces spec.build.source when set (used for preview builds). SourceOverride *decositesv1alpha1.DecoSpecBuildSource // BuilderImage is the platform default. spec.build.builder in the CR takes precedence when set. @@ -78,9 +76,11 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { {Name: "CF_ACCOUNT_ID", Value: opts.CfAccountId}, {Name: "CLOUDFLARE_API_TOKEN", Value: opts.CfApiToken}, {Name: "LOGS_UPLOAD_URL", Value: opts.presignedURLs.LogsUpload}, - {Name: "CACHE_DOWNLOAD_URL", Value: opts.presignedURLs.CacheDownload}, - {Name: "CACHE_UPLOAD_URL", Value: opts.presignedURLs.CacheUpload}, - {Name: "STATE_DOWNLOAD_URL", Value: opts.presignedURLs.StateDownload}, + {Name: "S3_ARTIFACTS_BUCKET", Value: opts.S3.ArtifactsBucket}, + {Name: "S3_STATE_BUCKET", Value: opts.S3.StateBucket}, + {Name: "S3_REGION", Value: opts.S3.Region}, + {Name: "S3_ACCESS_KEY_ID", Value: opts.S3.AccessKeyID}, + {Name: "S3_SECRET_ACCESS_KEY", Value: opts.S3.SecretAccessKey}, } if src.BranchRef != "" { env = append(env, corev1.EnvVar{Name: "BRANCH_REF", Value: src.BranchRef}) @@ -206,6 +206,7 @@ func (b *cfWorkersBuilder) NewJob(ctx context.Context, deco *decositesv1alpha1.D CfApiToken: b.cfg.CfApiToken, CfAccountId: b.cfg.CfAccountId, presignedURLs: urls, + S3: b.cfg.S3, SourceOverride: &source, BuilderImage: b.cfg.BuilderImage, TTLSeconds: b.cfg.TTLSeconds, diff --git a/internal/build/s3presign.go b/internal/build/s3presign.go index 5c1c907..961ecaa 100644 --- a/internal/build/s3presign.go +++ b/internal/build/s3presign.go @@ -19,12 +19,11 @@ type S3Config struct { AccessKeyID string SecretAccessKey string LogsBucket string // bucket for build logs - ArtifactsBucket string // bucket for build artifacts - StateBucket string // bucket for per-site state (defaults to ArtifactsBucket) + ArtifactsBucket string // bucket for npm cache + StateBucket string // bucket for per-site state (defaults to deco-admin-states) } -// GeneratepresignedURLs generates all presigned URLs the build job needs. -// Mirrors generatePresignedUrls() in the admin's build.ts. +// generatePresignedURLs generates the presigned URLs the build job needs. func generatePresignedURLs(ctx context.Context, cfg S3Config, site, jobName string) (presignedURLs, error) { awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(cfg.Region), @@ -46,40 +45,7 @@ func generatePresignedURLs(ctx context.Context, cfg S3Config, site, jobName stri return presignedURLs{}, fmt.Errorf("presigning logs upload: %w", err) } - cacheKey := fmt.Sprintf("%s/npm-cache.tar.zst", site) - - cacheDownload, err := presigner.PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(cfg.ArtifactsBucket), - Key: aws.String(cacheKey), - }, s3.WithPresignExpires(presignExpiry)) - if err != nil { - return presignedURLs{}, fmt.Errorf("presigning cache download: %w", err) - } - - cacheUpload, err := presigner.PresignPutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(cfg.ArtifactsBucket), - Key: aws.String(cacheKey), - }, s3.WithPresignExpires(presignExpiry)) - if err != nil { - return presignedURLs{}, fmt.Errorf("presigning cache upload: %w", err) - } - - stateBucket := cfg.StateBucket - if stateBucket == "" { - stateBucket = "deco-admin-states" - } - stateDownload, err := presigner.PresignGetObject(ctx, &s3.GetObjectInput{ - Bucket: aws.String(stateBucket), - Key: aws.String(fmt.Sprintf("site-state/%s/state.json", site)), - }, s3.WithPresignExpires(presignExpiry)) - if err != nil { - return presignedURLs{}, fmt.Errorf("presigning state download: %w", err) - } - return presignedURLs{ - LogsUpload: logsUpload.URL, - CacheDownload: cacheDownload.URL, - CacheUpload: cacheUpload.URL, - StateDownload: stateDownload.URL, + LogsUpload: logsUpload.URL, }, nil } From 2d8b0225c9945285e6818093d591bcd004548e61 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 19:26:06 -0300 Subject: [PATCH 04/11] refactor(cfworkers): remove presigned URLs entirely, builder uses S3 credentials directly --- internal/build/cfworkers.go | 26 +++++++++---------- internal/build/s3presign.go | 51 ------------------------------------- 2 files changed, 12 insertions(+), 65 deletions(-) delete mode 100644 internal/build/s3presign.go diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index c061a7e..b9104ee 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -21,9 +21,14 @@ func JobName(commitSha, site string) string { return fmt.Sprintf("build-%x", h[:4]) } -// presignedURLs are the S3 presigned URLs the build job needs. -type presignedURLs struct { - LogsUpload string +// S3Config holds the AWS credentials and bucket names for the build job. +type S3Config struct { + Region string + AccessKeyID string + SecretAccessKey string + LogsBucket string + ArtifactsBucket string + StateBucket string } // cfWorkersJobOpts are the inputs for NewJob. @@ -33,7 +38,6 @@ type cfWorkersJobOpts struct { GithubToken string CfApiToken string CfAccountId string - presignedURLs presignedURLs S3 S3Config // SourceOverride replaces spec.build.source when set (used for preview builds). SourceOverride *decositesv1alpha1.DecoSpecBuildSource @@ -75,7 +79,7 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { {Name: "IS_PRODUCTION", Value: isProduction}, {Name: "CF_ACCOUNT_ID", Value: opts.CfAccountId}, {Name: "CLOUDFLARE_API_TOKEN", Value: opts.CfApiToken}, - {Name: "LOGS_UPLOAD_URL", Value: opts.presignedURLs.LogsUpload}, + {Name: "S3_LOGS_BUCKET", Value: opts.S3.LogsBucket}, {Name: "S3_ARTIFACTS_BUCKET", Value: opts.S3.ArtifactsBucket}, {Name: "S3_STATE_BUCKET", Value: opts.S3.StateBucket}, {Name: "S3_REGION", Value: opts.S3.Region}, @@ -185,27 +189,21 @@ func CfWorkersConfigFromEnv() CfWorkersConfig { } type cfWorkersBuilder struct { - cfg CfWorkersConfig - presignFn func(ctx context.Context, cfg S3Config, site, jobName string) (presignedURLs, error) + cfg CfWorkersConfig } // NewCloudflareFactory returns a Builder for spec.serving.type = "cloudflare-worker". func NewCloudflareFactory(cfg CfWorkersConfig) Builder { - return &cfWorkersBuilder{cfg: cfg, presignFn: generatePresignedURLs} + return &cfWorkersBuilder{cfg: cfg} } -func (b *cfWorkersBuilder) NewJob(ctx context.Context, deco *decositesv1alpha1.Deco, jobName string, source decositesv1alpha1.DecoSpecBuildSource) (*batchv1.Job, error) { - urls, err := b.presignFn(ctx, b.cfg.S3, deco.Spec.Site, jobName) - if err != nil { - return nil, fmt.Errorf("generating presigned URLs: %w", err) - } +func (b *cfWorkersBuilder) NewJob(_ context.Context, deco *decositesv1alpha1.Deco, jobName string, source decositesv1alpha1.DecoSpecBuildSource) (*batchv1.Job, error) { return newCfWorkersJob(cfWorkersJobOpts{ Deco: deco, JobName: jobName, GithubToken: b.cfg.GithubToken, CfApiToken: b.cfg.CfApiToken, CfAccountId: b.cfg.CfAccountId, - presignedURLs: urls, S3: b.cfg.S3, SourceOverride: &source, BuilderImage: b.cfg.BuilderImage, diff --git a/internal/build/s3presign.go b/internal/build/s3presign.go deleted file mode 100644 index 961ecaa..0000000 --- a/internal/build/s3presign.go +++ /dev/null @@ -1,51 +0,0 @@ -package build - -import ( - "context" - "fmt" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" -) - -const presignExpiry = time.Hour - -// S3Config holds the AWS credentials and bucket names used to generate presigned URLs. -type S3Config struct { - Region string - AccessKeyID string - SecretAccessKey string - LogsBucket string // bucket for build logs - ArtifactsBucket string // bucket for npm cache - StateBucket string // bucket for per-site state (defaults to deco-admin-states) -} - -// generatePresignedURLs generates the presigned URLs the build job needs. -func generatePresignedURLs(ctx context.Context, cfg S3Config, site, jobName string) (presignedURLs, error) { - awsCfg, err := config.LoadDefaultConfig(ctx, - config.WithRegion(cfg.Region), - config.WithCredentialsProvider( - credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), - ), - ) - if err != nil { - return presignedURLs{}, fmt.Errorf("loading aws config: %w", err) - } - - presigner := s3.NewPresignClient(s3.NewFromConfig(awsCfg)) - - logsUpload, err := presigner.PresignPutObject(ctx, &s3.PutObjectInput{ - Bucket: aws.String(cfg.LogsBucket), - Key: aws.String(fmt.Sprintf("%s/%s.log", site, jobName)), - }, s3.WithPresignExpires(presignExpiry)) - if err != nil { - return presignedURLs{}, fmt.Errorf("presigning logs upload: %w", err) - } - - return presignedURLs{ - LogsUpload: logsUpload.URL, - }, nil -} From 4c04e553a00d8c9c29a32d0b07d4b1be7041cc21 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 20:19:58 -0300 Subject: [PATCH 05/11] feat(helm): add S3_STATE_BUCKET to cfworkers builder env via helm generator --- chart/templates/deployment-operator-controller-manager.yaml | 2 +- hack/helm-generator/main.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index 7aedac4..8173b63 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -31,7 +31,7 @@ spec: command: - /manager image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket }} + {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN diff --git a/hack/helm-generator/main.go b/hack/helm-generator/main.go index 7fa0078..3d74bff 100644 --- a/hack/helm-generator/main.go +++ b/hack/helm-generator/main.go @@ -188,7 +188,7 @@ func addEnvVarsToDeployment(templatesDir string) error { contentStr := string(content) // Find the image line and add env vars after it - envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket }} + envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -261,6 +261,10 @@ func addEnvVarsToDeployment(templatesDir string) error { - name: S3_LOGS_BUCKET value: {{ .logsBucket | quote }} {{- end }} + {{- if .stateBucket }} + - name: S3_STATE_BUCKET + value: {{ .stateBucket | quote }} + {{- end }} {{- end }} {{- end }}` From 5d31d02d8fa824f17ca78561ae5d0082bce96d01 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 20:50:07 -0300 Subject: [PATCH 06/11] fix(cfworkers): gofmt struct field alignment --- internal/build/cfworkers.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index b9104ee..ae29dd0 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -33,12 +33,12 @@ type S3Config struct { // cfWorkersJobOpts are the inputs for NewJob. type cfWorkersJobOpts struct { - Deco *decositesv1alpha1.Deco - JobName string - GithubToken string - CfApiToken string - CfAccountId string - S3 S3Config + Deco *decositesv1alpha1.Deco + JobName string + GithubToken string + CfApiToken string + CfAccountId string + S3 S3Config // SourceOverride replaces spec.build.source when set (used for preview builds). SourceOverride *decositesv1alpha1.DecoSpecBuildSource // BuilderImage is the platform default. spec.build.builder in the CR takes precedence when set. From 9b1b1a839cd40bce5e9eeb512f610f0cee090121 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 20:55:58 -0300 Subject: [PATCH 07/11] fix(helm): add stateBucket to chart values.yaml --- chart/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/values.yaml b/chart/values.yaml index 54f5f05..335d629 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -106,6 +106,7 @@ s3: existingSecret: "" # Secret with s3-access-key-id, s3-secret-access-key region: "" # AWS region (e.g. us-east-1) logsBucket: "" # S3 bucket for build logs + stateBucket: "" # S3 bucket for site state # Name overrides nameOverride: "" From 32121dec96abf5d5094ba8ba862719ab40f17947 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 21:34:31 -0300 Subject: [PATCH 08/11] feat(cfworkers): use Pod Identity for builder S3 access Remove static S3 credentials from builder jobs. The operator now creates a ServiceAccount (deco-operator-builders) in the site namespace before each job, and sets serviceAccountName on the pod spec. Credentials are provided via EKS Pod Identity, so S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are no longer passed. Also adds build.serviceAccount helm value and BUILD_SERVICE_ACCOUNT env var. --- .../clusterrole-operator-manager-role.yaml | 7 +++ ...eployment-operator-controller-manager.yaml | 20 +++---- chart/values.yaml | 9 ++- config/rbac/role.yaml | 7 +++ hack/helm-generator/main.go | 20 +++---- internal/build/cfworkers.go | 57 ++++++++++--------- internal/controller/deco_controller.go | 19 +++++++ 7 files changed, 82 insertions(+), 57 deletions(-) diff --git a/chart/templates/clusterrole-operator-manager-role.yaml b/chart/templates/clusterrole-operator-manager-role.yaml index 79c3477..08f004c 100644 --- a/chart/templates/clusterrole-operator-manager-role.yaml +++ b/chart/templates/clusterrole-operator-manager-role.yaml @@ -33,6 +33,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create - apiGroups: - "" resources: diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index 8173b63..27cb13f 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -31,7 +31,7 @@ spec: command: - /manager image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket }} + {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -84,18 +84,6 @@ spec: {{- end }} {{- end }} {{- with .Values.s3 }} - {{- if .existingSecret }} - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .existingSecret | quote }} - key: s3-access-key-id - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .existingSecret | quote }} - key: s3-secret-access-key - {{- end }} {{- if .region }} - name: S3_REGION value: {{ .region | quote }} @@ -109,6 +97,12 @@ spec: value: {{ .stateBucket | quote }} {{- end }} {{- end }} + {{- with .Values.build }} + {{- if .serviceAccount }} + - name: BUILD_SERVICE_ACCOUNT + value: {{ .serviceAccount | quote }} + {{- end }} + {{- end }} {{- end }} livenessProbe: httpGet: diff --git a/chart/values.yaml b/chart/values.yaml index 335d629..37dcd29 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -101,12 +101,15 @@ cfworkers: builderImage: "" # Builder container image (repository:tag) artifactsBucket: "" # S3 bucket for build artifacts (cfworkers-specific) -# S3 credentials and config — shared across all build platforms +# S3 config — shared across all build platforms (credentials via Pod Identity) s3: - existingSecret: "" # Secret with s3-access-key-id, s3-secret-access-key region: "" # AWS region (e.g. us-east-1) logsBucket: "" # S3 bucket for build logs - stateBucket: "" # S3 bucket for site state + stateBucket: "" # S3 bucket for site state + +# Build job config — shared across all build platforms +build: + serviceAccount: "" # K8s ServiceAccount for builder pods (Pod Identity) # Name overrides nameOverride: "" diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2439d35..8117931 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -34,6 +34,13 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - get + - create - apiGroups: - "" resources: diff --git a/hack/helm-generator/main.go b/hack/helm-generator/main.go index 3d74bff..a886c71 100644 --- a/hack/helm-generator/main.go +++ b/hack/helm-generator/main.go @@ -188,7 +188,7 @@ func addEnvVarsToDeployment(templatesDir string) error { contentStr := string(content) // Find the image line and add env vars after it - envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.existingSecret .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket }} + envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -241,18 +241,6 @@ func addEnvVarsToDeployment(templatesDir string) error { {{- end }} {{- end }} {{- with .Values.s3 }} - {{- if .existingSecret }} - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ .existingSecret | quote }} - key: s3-access-key-id - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .existingSecret | quote }} - key: s3-secret-access-key - {{- end }} {{- if .region }} - name: S3_REGION value: {{ .region | quote }} @@ -266,6 +254,12 @@ func addEnvVarsToDeployment(templatesDir string) error { value: {{ .stateBucket | quote }} {{- end }} {{- end }} + {{- with .Values.build }} + {{- if .serviceAccount }} + - name: BUILD_SERVICE_ACCOUNT + value: {{ .serviceAccount | quote }} + {{- end }} + {{- end }} {{- end }}` re := regexp.MustCompile(`(?m)( image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}")`) diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index ae29dd0..0737784 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -21,11 +21,10 @@ func JobName(commitSha, site string) string { return fmt.Sprintf("build-%x", h[:4]) } -// S3Config holds the AWS credentials and bucket names for the build job. +// S3Config holds the bucket names and region for the build job. +// Credentials are provided via Pod Identity (no static keys needed). type S3Config struct { Region string - AccessKeyID string - SecretAccessKey string LogsBucket string ArtifactsBucket string StateBucket string @@ -43,6 +42,8 @@ type cfWorkersJobOpts struct { SourceOverride *decositesv1alpha1.DecoSpecBuildSource // BuilderImage is the platform default. spec.build.builder in the CR takes precedence when set. BuilderImage string + // BuilderServiceAccount is the K8s ServiceAccount the job pod runs as (for Pod Identity). + BuilderServiceAccount string // TTLSeconds controls how long the Job is kept after completion. TTLSeconds int32 } @@ -83,8 +84,6 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { {Name: "S3_ARTIFACTS_BUCKET", Value: opts.S3.ArtifactsBucket}, {Name: "S3_STATE_BUCKET", Value: opts.S3.StateBucket}, {Name: "S3_REGION", Value: opts.S3.Region}, - {Name: "S3_ACCESS_KEY_ID", Value: opts.S3.AccessKeyID}, - {Name: "S3_SECRET_ACCESS_KEY", Value: opts.S3.SecretAccessKey}, } if src.BranchRef != "" { env = append(env, corev1.EnvVar{Name: "BRANCH_REF", Value: src.BranchRef}) @@ -133,7 +132,8 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { TTLSecondsAfterFinished: &ttl, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyNever, + RestartPolicy: corev1.RestartPolicyNever, + ServiceAccountName: opts.BuilderServiceAccount, Containers: []corev1.Container{ { Name: "builder", @@ -161,26 +161,26 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { // CfWorkersConfig holds all configuration the Cloudflare Workers builder needs. type CfWorkersConfig struct { - CfApiToken string - CfAccountId string - GithubToken string - BuilderImage string - TTLSeconds int32 - S3 S3Config + CfApiToken string + CfAccountId string + GithubToken string + BuilderImage string + BuilderServiceAccount string + TTLSeconds int32 + S3 S3Config } // CfWorkersConfigFromEnv reads CfWorkersConfig from standard environment variables. func CfWorkersConfigFromEnv() CfWorkersConfig { return CfWorkersConfig{ - CfApiToken: os.Getenv("CLOUDFLARE_API_WORKERS_TOKEN"), - CfAccountId: os.Getenv("CLOUDFLARE_ACCOUNT_ID"), - GithubToken: os.Getenv("GITHUB_TOKEN"), - BuilderImage: os.Getenv("CFWORKERS_BUILDER_IMAGE"), - TTLSeconds: 10 * 60, + CfApiToken: os.Getenv("CLOUDFLARE_API_WORKERS_TOKEN"), + CfAccountId: os.Getenv("CLOUDFLARE_ACCOUNT_ID"), + GithubToken: os.Getenv("GITHUB_TOKEN"), + BuilderImage: os.Getenv("CFWORKERS_BUILDER_IMAGE"), + BuilderServiceAccount: os.Getenv("BUILD_SERVICE_ACCOUNT"), + TTLSeconds: 10 * 60, S3: S3Config{ Region: os.Getenv("S3_REGION"), - AccessKeyID: os.Getenv("S3_ACCESS_KEY_ID"), - SecretAccessKey: os.Getenv("S3_SECRET_ACCESS_KEY"), LogsBucket: os.Getenv("S3_LOGS_BUCKET"), ArtifactsBucket: os.Getenv("S3_ARTIFACTS_BUCKET"), StateBucket: os.Getenv("S3_STATE_BUCKET"), @@ -199,14 +199,15 @@ func NewCloudflareFactory(cfg CfWorkersConfig) Builder { func (b *cfWorkersBuilder) NewJob(_ context.Context, deco *decositesv1alpha1.Deco, jobName string, source decositesv1alpha1.DecoSpecBuildSource) (*batchv1.Job, error) { return newCfWorkersJob(cfWorkersJobOpts{ - Deco: deco, - JobName: jobName, - GithubToken: b.cfg.GithubToken, - CfApiToken: b.cfg.CfApiToken, - CfAccountId: b.cfg.CfAccountId, - S3: b.cfg.S3, - SourceOverride: &source, - BuilderImage: b.cfg.BuilderImage, - TTLSeconds: b.cfg.TTLSeconds, + Deco: deco, + JobName: jobName, + GithubToken: b.cfg.GithubToken, + CfApiToken: b.cfg.CfApiToken, + CfAccountId: b.cfg.CfAccountId, + S3: b.cfg.S3, + SourceOverride: &source, + BuilderImage: b.cfg.BuilderImage, + BuilderServiceAccount: b.cfg.BuilderServiceAccount, + TTLSeconds: b.cfg.TTLSeconds, }), nil } diff --git a/internal/controller/deco_controller.go b/internal/controller/deco_controller.go index 9c89ebf..5996232 100644 --- a/internal/controller/deco_controller.go +++ b/internal/controller/deco_controller.go @@ -38,6 +38,7 @@ type DecoReconciler struct { // +kubebuilder:rbac:groups=deco.sites,resources=decos/status,verbs=get;update;patch // +kubebuilder:rbac:groups=deco.sites,resources=decos/finalizers,verbs=update // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete +// +kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;create func (r *DecoReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -249,6 +250,12 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. return fmt.Errorf("building job spec: %w", err) } + if sa := job.Spec.Template.Spec.ServiceAccountName; sa != "" { + if err := r.ensureServiceAccount(ctx, deco.Namespace, sa); err != nil { + return fmt.Errorf("ensuring service account %q: %w", sa, err) + } + } + if err := controllerutil.SetControllerReference(deco, job, r.Scheme); err != nil { return fmt.Errorf("setting owner reference: %w", err) } @@ -259,6 +266,18 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. return nil } +// ensureServiceAccount creates the ServiceAccount in the given namespace if it doesn't exist. +func (r *DecoReconciler) ensureServiceAccount(ctx context.Context, namespace, name string) error { + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + } + err := r.Create(ctx, sa) + if errors.IsAlreadyExists(err) { + return nil + } + return err +} + func buildPhaseFromJob(job *batchv1.Job) string { for _, c := range job.Status.Conditions { if c.Status != corev1.ConditionTrue { From 0f3f646591acc317ea2d65b5bd684facec971437 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 22:05:42 -0300 Subject: [PATCH 09/11] feat(irsa): annotate builder ServiceAccount with IAM role ARN for IRSA Add BUILD_ROLE_ARN env var. The operator sets eks.amazonaws.com/role-arn on the ServiceAccount via CreateOrUpdate so IRSA credentials are injected into builder pods automatically. --- ...eployment-operator-controller-manager.yaml | 6 ++++- chart/values.yaml | 3 ++- cmd/main.go | 7 +++--- hack/helm-generator/main.go | 6 ++++- internal/controller/deco_controller.go | 22 ++++++++++++------- 5 files changed, 30 insertions(+), 14 deletions(-) diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index 27cb13f..8091fa5 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -31,7 +31,7 @@ spec: command: - /manager image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount }} + {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -102,6 +102,10 @@ spec: - name: BUILD_SERVICE_ACCOUNT value: {{ .serviceAccount | quote }} {{- end }} + {{- if .roleArn }} + - name: BUILD_ROLE_ARN + value: {{ .roleArn | quote }} + {{- end }} {{- end }} {{- end }} livenessProbe: diff --git a/chart/values.yaml b/chart/values.yaml index 37dcd29..c2009bc 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -109,7 +109,8 @@ s3: # Build job config — shared across all build platforms build: - serviceAccount: "" # K8s ServiceAccount for builder pods (Pod Identity) + serviceAccount: "" # K8s ServiceAccount for builder pods (IRSA) + roleArn: "" # IAM role ARN for IRSA annotation on the ServiceAccount # Name overrides nameOverride: "" diff --git a/cmd/main.go b/cmd/main.go index 5661e13..4974f3f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -333,9 +333,10 @@ func main() { registry.Register("cloudflare-worker", build.NewCloudflareFactory(build.CfWorkersConfigFromEnv())) if err := (&controller.DecoReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Builder: registry, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Builder: registry, + BuilderRoleArn: os.Getenv("BUILD_ROLE_ARN"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Deco") os.Exit(1) diff --git a/hack/helm-generator/main.go b/hack/helm-generator/main.go index a886c71..ed843a8 100644 --- a/hack/helm-generator/main.go +++ b/hack/helm-generator/main.go @@ -188,7 +188,7 @@ func addEnvVarsToDeployment(templatesDir string) error { contentStr := string(content) // Find the image line and add env vars after it - envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount }} + envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -259,6 +259,10 @@ func addEnvVarsToDeployment(templatesDir string) error { - name: BUILD_SERVICE_ACCOUNT value: {{ .serviceAccount | quote }} {{- end }} + {{- if .roleArn }} + - name: BUILD_ROLE_ARN + value: {{ .roleArn | quote }} + {{- end }} {{- end }} {{- end }}` diff --git a/internal/controller/deco_controller.go b/internal/controller/deco_controller.go index 5996232..a03092d 100644 --- a/internal/controller/deco_controller.go +++ b/internal/controller/deco_controller.go @@ -30,8 +30,9 @@ const ( // DecoReconciler reconciles Deco objects. type DecoReconciler struct { client.Client - Scheme *runtime.Scheme - Builder build.Builder + Scheme *runtime.Scheme + Builder build.Builder + BuilderRoleArn string } // +kubebuilder:rbac:groups=deco.sites,resources=decos,verbs=get;list;watch;create;update;patch;delete @@ -251,7 +252,7 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. } if sa := job.Spec.Template.Spec.ServiceAccountName; sa != "" { - if err := r.ensureServiceAccount(ctx, deco.Namespace, sa); err != nil { + if err := r.ensureServiceAccount(ctx, deco.Namespace, sa, r.BuilderRoleArn); err != nil { return fmt.Errorf("ensuring service account %q: %w", sa, err) } } @@ -266,15 +267,20 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. return nil } -// ensureServiceAccount creates the ServiceAccount in the given namespace if it doesn't exist. -func (r *DecoReconciler) ensureServiceAccount(ctx context.Context, namespace, name string) error { +// ensureServiceAccount creates or updates the ServiceAccount with the IRSA annotation if roleArn is set. +func (r *DecoReconciler) ensureServiceAccount(ctx context.Context, namespace, name, roleArn string) error { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, } - err := r.Create(ctx, sa) - if errors.IsAlreadyExists(err) { + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { + if roleArn != "" { + if sa.Annotations == nil { + sa.Annotations = map[string]string{} + } + sa.Annotations["eks.amazonaws.com/role-arn"] = roleArn + } return nil - } + }) return err } From c782a0bc8b8acf0a55c5a7d80acef77edae23378 Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 22:25:02 -0300 Subject: [PATCH 10/11] feat(build): support nodeSelector, tolerations and N SA annotations for builder pods Allows targeting a specific node pool and scheduling on tainted nodes, configurable as operator-level defaults (Helm values / env vars) with per-site override via spec.build.nodeSelector and spec.build.tolerations. Also replaces the single BuilderRoleArn with a generic BuilderSAAnnotations map so any number of ServiceAccount annotations can be injected. Co-Authored-By: Claude Sonnet 4.6 --- api/v1alpha1/deco_types.go | 14 ++++- api/v1alpha1/zz_generated.deepcopy.go | 15 +++++ ...eployment-operator-controller-manager.yaml | 10 +++- chart/values.yaml | 2 + cmd/main.go | 12 ++-- internal/build/cfworkers.go | 60 +++++++++++++++++++ internal/controller/deco_controller.go | 18 +++--- 7 files changed, 117 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/deco_types.go b/api/v1alpha1/deco_types.go index d9865a7..74a403c 100644 --- a/api/v1alpha1/deco_types.go +++ b/api/v1alpha1/deco_types.go @@ -1,6 +1,9 @@ package v1alpha1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // DecoSpec defines the desired state of a Deco workload. type DecoSpec struct { @@ -82,6 +85,15 @@ type DecoSpecBuild struct { // +optional // +kubebuilder:validation:Minimum=0 TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"` + + // NodeSelector constrains the build Job pods to nodes matching all the specified labels. + // Use this to target a specific node pool (e.g. cloud.google.com/gke-nodepool: build-pool). + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Tolerations are applied to the build Job pods, allowing them to be scheduled on tainted nodes. + // +optional + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } // DecoSpecBuildSource identifies the code revision to build. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a0d6a79..7fcf805 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -228,6 +229,20 @@ func (in *DecoSpecBuild) DeepCopyInto(out *DecoSpecBuild) { *out = new(int32) **out = **in } + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DecoSpecBuild. diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index 8091fa5..e8cf038 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -31,7 +31,7 @@ spec: command: - /manager image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn }} + {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -106,6 +106,14 @@ spec: - name: BUILD_ROLE_ARN value: {{ .roleArn | quote }} {{- end }} + {{- if .nodeSelector }} + - name: BUILD_NODE_SELECTOR + value: {{ $parts := list }}{{ range $k, $v := .nodeSelector }}{{ $parts = append $parts (printf "%s=%s" $k $v) }}{{ end }}{{ join "," $parts | quote }} + {{- end }} + {{- if .tolerations }} + - name: BUILD_TOLERATIONS + value: {{ .tolerations | toJson | quote }} + {{- end }} {{- end }} {{- end }} livenessProbe: diff --git a/chart/values.yaml b/chart/values.yaml index c2009bc..4ae2820 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -111,6 +111,8 @@ s3: build: serviceAccount: "" # K8s ServiceAccount for builder pods (IRSA) roleArn: "" # IAM role ARN for IRSA annotation on the ServiceAccount + nodeSelector: {} # Default nodeSelector for build pods (overridable per-site via spec.build.nodeSelector) + tolerations: [] # Default tolerations for build pods (overridable per-site via spec.build.tolerations) # Name overrides nameOverride: "" diff --git a/cmd/main.go b/cmd/main.go index 4974f3f..699153c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -332,11 +332,15 @@ func main() { registry := build.NewBuilderRegistry() registry.Register("cloudflare-worker", build.NewCloudflareFactory(build.CfWorkersConfigFromEnv())) + builderSAAnnotations := map[string]string{} + if roleArn := os.Getenv("BUILD_ROLE_ARN"); roleArn != "" { + builderSAAnnotations["eks.amazonaws.com/role-arn"] = roleArn + } if err := (&controller.DecoReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Builder: registry, - BuilderRoleArn: os.Getenv("BUILD_ROLE_ARN"), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Builder: registry, + BuilderSAAnnotations: builderSAAnnotations, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Deco") os.Exit(1) diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index 0737784..4f70e50 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -4,8 +4,10 @@ package build import ( "context" "crypto/sha256" + "encoding/json" "fmt" "os" + "strings" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -46,6 +48,10 @@ type cfWorkersJobOpts struct { BuilderServiceAccount string // TTLSeconds controls how long the Job is kept after completion. TTLSeconds int32 + // NodeSelector constrains build pods to nodes matching these labels. + NodeSelector map[string]string + // Tolerations applied to build pods. + Tolerations []corev1.Toleration } // newCfWorkersJob builds the batchv1.Job spec for a cfworkers build. @@ -117,6 +123,16 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { ttl = *spec.Build.TTLSecondsAfterFinished } + nodeSelector := opts.NodeSelector + if spec.Build != nil && len(spec.Build.NodeSelector) > 0 { + nodeSelector = spec.Build.NodeSelector + } + + tolerations := opts.Tolerations + if spec.Build != nil && len(spec.Build.Tolerations) > 0 { + tolerations = spec.Build.Tolerations + } + return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: opts.JobName, @@ -134,6 +150,8 @@ func newCfWorkersJob(opts cfWorkersJobOpts) *batchv1.Job { Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: opts.BuilderServiceAccount, + NodeSelector: nodeSelector, + Tolerations: tolerations, Containers: []corev1.Container{ { Name: "builder", @@ -168,6 +186,12 @@ type CfWorkersConfig struct { BuilderServiceAccount string TTLSeconds int32 S3 S3Config + // NodeSelector is the default nodeSelector applied to all build pods. + // spec.build.nodeSelector in the CR overrides this per-site. + NodeSelector map[string]string + // Tolerations is the default list of tolerations applied to all build pods. + // spec.build.tolerations in the CR overrides this per-site. + Tolerations []corev1.Toleration } // CfWorkersConfigFromEnv reads CfWorkersConfig from standard environment variables. @@ -179,6 +203,8 @@ func CfWorkersConfigFromEnv() CfWorkersConfig { BuilderImage: os.Getenv("CFWORKERS_BUILDER_IMAGE"), BuilderServiceAccount: os.Getenv("BUILD_SERVICE_ACCOUNT"), TTLSeconds: 10 * 60, + NodeSelector: parseNodeSelector(os.Getenv("BUILD_NODE_SELECTOR")), + Tolerations: parseTolerations(os.Getenv("BUILD_TOLERATIONS")), S3: S3Config{ Region: os.Getenv("S3_REGION"), LogsBucket: os.Getenv("S3_LOGS_BUCKET"), @@ -188,6 +214,38 @@ func CfWorkersConfigFromEnv() CfWorkersConfig { } } +// parseNodeSelector parses a comma-separated "key=value" string into a map. +// Empty or malformed pairs are silently skipped. +func parseNodeSelector(s string) map[string]string { + if s == "" { + return nil + } + m := map[string]string{} + for _, pair := range strings.Split(s, ",") { + k, v, ok := strings.Cut(pair, "=") + if ok && k != "" { + m[k] = v + } + } + if len(m) == 0 { + return nil + } + return m +} + +// parseTolerations unmarshals a JSON array of Toleration objects. +// Returns nil on empty input or parse error. +func parseTolerations(s string) []corev1.Toleration { + if s == "" { + return nil + } + var t []corev1.Toleration + if err := json.Unmarshal([]byte(s), &t); err != nil { + return nil + } + return t +} + type cfWorkersBuilder struct { cfg CfWorkersConfig } @@ -209,5 +267,7 @@ func (b *cfWorkersBuilder) NewJob(_ context.Context, deco *decositesv1alpha1.Dec BuilderImage: b.cfg.BuilderImage, BuilderServiceAccount: b.cfg.BuilderServiceAccount, TTLSeconds: b.cfg.TTLSeconds, + NodeSelector: b.cfg.NodeSelector, + Tolerations: b.cfg.Tolerations, }), nil } diff --git a/internal/controller/deco_controller.go b/internal/controller/deco_controller.go index a03092d..172d4b5 100644 --- a/internal/controller/deco_controller.go +++ b/internal/controller/deco_controller.go @@ -30,9 +30,9 @@ const ( // DecoReconciler reconciles Deco objects. type DecoReconciler struct { client.Client - Scheme *runtime.Scheme - Builder build.Builder - BuilderRoleArn string + Scheme *runtime.Scheme + Builder build.Builder + BuilderSAAnnotations map[string]string } // +kubebuilder:rbac:groups=deco.sites,resources=decos,verbs=get;list;watch;create;update;patch;delete @@ -252,7 +252,7 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. } if sa := job.Spec.Template.Spec.ServiceAccountName; sa != "" { - if err := r.ensureServiceAccount(ctx, deco.Namespace, sa, r.BuilderRoleArn); err != nil { + if err := r.ensureServiceAccount(ctx, deco.Namespace, sa, r.BuilderSAAnnotations); err != nil { return fmt.Errorf("ensuring service account %q: %w", sa, err) } } @@ -267,17 +267,19 @@ func (r *DecoReconciler) createJob(ctx context.Context, deco *decositesv1alpha1. return nil } -// ensureServiceAccount creates or updates the ServiceAccount with the IRSA annotation if roleArn is set. -func (r *DecoReconciler) ensureServiceAccount(ctx context.Context, namespace, name, roleArn string) error { +// ensureServiceAccount creates or updates the ServiceAccount merging the provided annotations. +func (r *DecoReconciler) ensureServiceAccount(ctx context.Context, namespace, name string, annotations map[string]string) error { sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, } _, err := controllerutil.CreateOrUpdate(ctx, r.Client, sa, func() error { - if roleArn != "" { + if len(annotations) > 0 { if sa.Annotations == nil { sa.Annotations = map[string]string{} } - sa.Annotations["eks.amazonaws.com/role-arn"] = roleArn + for k, v := range annotations { + sa.Annotations[k] = v + } } return nil }) From b5cc8aaae6e039f479e26f9644c63b58fb5fb82d Mon Sep 17 00:00:00 2001 From: igoramf Date: Thu, 7 May 2026 22:31:06 -0300 Subject: [PATCH 11/11] fix(helm): regenerate chart templates with nodeSelector, tolerations and SA annotations support Updates helm-generator to emit BUILD_NODE_SELECTOR and BUILD_TOLERATIONS env vars, fixes the $-variable corruption in ReplaceAllString by switching to strings.Replace, and switches nodeSelector env var to JSON format (consistent with tolerations). Regenerates CRDs and RBAC via controller-gen to include the new fields. Co-Authored-By: Claude Sonnet 4.6 --- .../clusterrole-operator-manager-role.yaml | 10 ++-- ...omresourcedefinition-decos.deco.sites.yaml | 47 +++++++++++++++++++ ...eployment-operator-controller-manager.yaml | 2 +- config/crd/bases/deco.sites_decos.yaml | 47 +++++++++++++++++++ config/rbac/role.yaml | 10 ++-- hack/helm-generator/main.go | 14 ++++-- internal/build/cfworkers.go | 12 ++--- 7 files changed, 120 insertions(+), 22 deletions(-) diff --git a/chart/templates/clusterrole-operator-manager-role.yaml b/chart/templates/clusterrole-operator-manager-role.yaml index 08f004c..4955b35 100644 --- a/chart/templates/clusterrole-operator-manager-role.yaml +++ b/chart/templates/clusterrole-operator-manager-role.yaml @@ -36,19 +36,19 @@ rules: - apiGroups: - "" resources: - - serviceaccounts + - secrets verbs: - - get - create + - get + - list + - watch - apiGroups: - "" resources: - - secrets + - serviceaccounts verbs: - create - get - - list - - watch - apiGroups: - batch resources: diff --git a/chart/templates/customresourcedefinition-decos.deco.sites.yaml b/chart/templates/customresourcedefinition-decos.deco.sites.yaml index 3bb2657..e6e2265 100644 --- a/chart/templates/customresourcedefinition-decos.deco.sites.yaml +++ b/chart/templates/customresourcedefinition-decos.deco.sites.yaml @@ -77,6 +77,13 @@ spec: - name type: object type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector constrains the build Job pods to nodes matching all the specified labels. + Use this to target a specific node pool (e.g. cloud.google.com/gke-nodepool: build-pool). + type: object secrets: description: |- Secrets are K8s Secrets whose keys are mounted as environment variables in the build Job. @@ -118,6 +125,46 @@ spec: required: - commitSha type: object + tolerations: + description: Tolerations are applied to the build Job pods, allowing + them to be scheduled on tainted nodes. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array ttlSecondsAfterFinished: description: |- TTLSecondsAfterFinished controls how long the Job is kept after completion. diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index e8cf038..8a1ba4d 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -108,7 +108,7 @@ spec: {{- end }} {{- if .nodeSelector }} - name: BUILD_NODE_SELECTOR - value: {{ $parts := list }}{{ range $k, $v := .nodeSelector }}{{ $parts = append $parts (printf "%s=%s" $k $v) }}{{ end }}{{ join "," $parts | quote }} + value: {{ .nodeSelector | toJson | quote }} {{- end }} {{- if .tolerations }} - name: BUILD_TOLERATIONS diff --git a/config/crd/bases/deco.sites_decos.yaml b/config/crd/bases/deco.sites_decos.yaml index c84c3f4..d2b912e 100644 --- a/config/crd/bases/deco.sites_decos.yaml +++ b/config/crd/bases/deco.sites_decos.yaml @@ -78,6 +78,13 @@ spec: - name type: object type: array + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector constrains the build Job pods to nodes matching all the specified labels. + Use this to target a specific node pool (e.g. cloud.google.com/gke-nodepool: build-pool). + type: object secrets: description: |- Secrets are K8s Secrets whose keys are mounted as environment variables in the build Job. @@ -119,6 +126,46 @@ spec: required: - commitSha type: object + tolerations: + description: Tolerations are applied to the build Job pods, allowing + them to be scheduled on tainted nodes. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array ttlSecondsAfterFinished: description: |- TTLSecondsAfterFinished controls how long the Job is kept after completion. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 8117931..728eecd 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -37,19 +37,19 @@ rules: - apiGroups: - "" resources: - - serviceaccounts + - secrets verbs: - - get - create + - get + - list + - watch - apiGroups: - "" resources: - - secrets + - serviceaccounts verbs: - create - get - - list - - watch - apiGroups: - batch resources: diff --git a/hack/helm-generator/main.go b/hack/helm-generator/main.go index ed843a8..a30b296 100644 --- a/hack/helm-generator/main.go +++ b/hack/helm-generator/main.go @@ -188,7 +188,7 @@ func addEnvVarsToDeployment(templatesDir string) error { contentStr := string(content) // Find the image line and add env vars after it - envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn }} + envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -263,11 +263,19 @@ func addEnvVarsToDeployment(templatesDir string) error { - name: BUILD_ROLE_ARN value: {{ .roleArn | quote }} {{- end }} + {{- if .nodeSelector }} + - name: BUILD_NODE_SELECTOR + value: {{ .nodeSelector | toJson | quote }} + {{- end }} + {{- if .tolerations }} + - name: BUILD_TOLERATIONS + value: {{ .tolerations | toJson | quote }} + {{- end }} {{- end }} {{- end }}` - re := regexp.MustCompile(`(?m)( image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}")`) - contentStr = re.ReplaceAllString(contentStr, "$1\n"+envBlock) + imageLine := ` image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"` + contentStr = strings.Replace(contentStr, imageLine, imageLine+"\n"+envBlock, 1) return os.WriteFile(deploymentFile, []byte(contentStr), 0644) } diff --git a/internal/build/cfworkers.go b/internal/build/cfworkers.go index 4f70e50..217799d 100644 --- a/internal/build/cfworkers.go +++ b/internal/build/cfworkers.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "os" - "strings" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -214,18 +213,15 @@ func CfWorkersConfigFromEnv() CfWorkersConfig { } } -// parseNodeSelector parses a comma-separated "key=value" string into a map. -// Empty or malformed pairs are silently skipped. +// parseNodeSelector parses a JSON object string into a map. +// Returns nil on empty input or parse error. func parseNodeSelector(s string) map[string]string { if s == "" { return nil } m := map[string]string{} - for _, pair := range strings.Split(s, ",") { - k, v, ok := strings.Cut(pair, "=") - if ok && k != "" { - m[k] = v - } + if err := json.Unmarshal([]byte(s), &m); err != nil { + return nil } if len(m) == 0 { return nil