feat: allow recreating a resource when a server-side apply is rejected

.github/workflows/codeql.yaml

@@ -0,0 +1,51 @@
+name: CodeQL
+on:
+  push:
+    branches:
+    - 'main'
+  pull_request:
+    branches:
+    - 'main'
+  schedule:
+  - cron: '21 3 * * 3'
+
+permissions: read-all
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['go']
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+    - name: Set up Go
+      uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
+      with:
+        go-version: 1.26.5
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Build
+      # Autobuild's heuristic only traced a fraction of the module (79 of 189 files per the
+      # code-scanning diagnostic). Build the whole module explicitly so CodeQL sees every package.
+      run: go build ./...
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+      with:
+        category: "/language:${{ matrix.language }}"

.github/workflows/scorecard.yaml

@@ -0,0 +1,45 @@
+name: Scorecard supply-chain security
+on:
+  branch_protection_rule:
+  schedule:
+  - cron: '30 2 * * 1'
+  push:
+    branches:
+    - 'main'
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+      contents: read
+      actions: read
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      with:
+        persist-credentials: false
+
+    - name: Run analysis
+      uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+      with:
+        results_file: results.sarif
+        results_format: sarif
+        publish_results: true
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: SARIF file
+        path: results.sarif
+        retention-days: 5
+
+    - name: Upload to code-scanning
+      uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
+      with:
+        sarif_file: results.sarif

README.md

@@ -1,5 +1,6 @@
 [![CI](https://github.com/projectsveltos/libsveltos/actions/workflows/main.yaml/badge.svg)](https://github.com/projectsveltos/libsveltos/actions)
-[![Go Report Card](https://goreportcard.com/badge/github.com/projectsveltos/libsveltos)](https://goreportcard.com/report/github.com/projectsveltos/libsveltos)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/projectsveltos/libsveltos/badge)](https://scorecard.dev/viewer/?uri=github.com/projectsveltos/libsveltos)
+[![CodeQL](https://github.com/projectsveltos/libsveltos/actions/workflows/codeql.yaml/badge.svg)](https://github.com/projectsveltos/libsveltos/actions/workflows/codeql.yaml)
 [![Release](https://img.shields.io/github/v/release/projectsveltos/libsveltos)](https://github.com/projectsveltos/libsveltos/releases)
 [![License](https://img.shields.io/badge/license-Apache-blue.svg)](LICENSE)
 [![Slack](https://img.shields.io/badge/join%20slack-%23projectsveltos-brighteen)](https://join.slack.com/t/projectsveltos/shared_invite/zt-1hraownbr-W8NTs6LTimxLPB8Erj8Q6Q)
@@ -22,7 +23,7 @@
 ## What this repository is
 Libraries containing all pieces common across different repositories
 
-## Contributing 
+## Contributing
 
 ❤️ Your contributions are always welcome! If you want to contribute, have questions, noticed any bug or want to get the latest project news, you can connect with us in the following ways:
 

SECURITY.md

@@ -0,0 +1,47 @@
+# Security Policy
+
+## Supported Versions
+
+We release security fixes for the latest minor version. We encourage all users to stay on the latest release.
+
+| Version        | Supported          |
+|----------------|--------------------|
+| latest release | :white_check_mark: |
+| older releases | :x:                |
+
+## Reporting a Vulnerability
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+If you believe you have found a security vulnerability in any Sveltos repository, please report it responsibly by sending an email to:
+
+**support@projectsveltos.io**
+
+Please include as much of the following information as possible to help us understand and resolve the issue quickly:
+
+- A description of the vulnerability and its potential impact
+- The affected component(s) and version(s)
+- Step-by-step instructions to reproduce the issue
+- Any proof-of-concept or exploit code (if applicable)
+- Suggested remediation (if any)
+
+## Response Process
+
+- You will receive an acknowledgement within **2 business days**
+- We will investigate and keep you informed of our progress
+- Once the issue is confirmed, we will work on a fix and coordinate a release
+- We will publicly disclose the vulnerability after a fix is available, giving you credit unless you prefer to remain anonymous
+
+## Scope
+
+This policy covers all projects under the [projectsveltos](https://github.com/projectsveltos) GitHub organization.
+
+## Out of Scope
+
+- Vulnerabilities in dependencies (please report those to the upstream project)
+- Issues in non-production branches or unreleased code
+- Social engineering attacks
+
+## Thank You
+
+We appreciate responsible disclosure and the work of the security community in keeping Sveltos and its users safe.

api/v1beta1/configurationbundle_type.go

@@ -112,6 +112,14 @@ type ConfigurationBundleSpec struct {
 	// +kubebuilder:default:=false
 	// +optional
 	SkipNamespaceCreation bool `json:"skipNamespaceCreation,omitempty"`
+
+	// Force indicates whether Sveltos should delete and recreate a resource defined in this
+	// PolicyRef/KustomizationRef when an update is rejected with an error that only a
+	// delete+recreate can resolve (eg an invalid combination of fields, or a field enforced
+	// as immutable). By default, such errors are surfaced instead of recreating the resource.
+	// +kubebuilder:default:=false
+	// +optional
+	Force bool `json:"force,omitempty"`
 }
 
 type ConfigurationBundleStatus struct {

config/crd/bases/lib.projectsveltos.io_configurationbundles.yaml

@@ -39,6 +39,14 @@ spec:
             type: object
           spec:
             properties:
+              force:
+                default: false
+                description: |-
+                  Force indicates whether Sveltos should delete and recreate a resource defined in this
+                  PolicyRef/KustomizationRef when an update is rejected with an error that only a
+                  delete+recreate can resolve (eg an invalid combination of fields, or a field enforced
+                  as immutable). By default, such errors are surfaced instead of recreating the resource.
+                type: boolean
               helmChartVersion:
                 description: |-
                   HelmChartVersion indicates the chart version of the Helm release

lib/crd/configurationbundles.go

@@ -57,6 +57,14 @@ spec:
             type: object
           spec:
             properties:
+              force:
+                default: false
+                description: |-
+                  Force indicates whether Sveltos should delete and recreate a resource defined in this
+                  PolicyRef/KustomizationRef when an update is rejected with an error that only a
+                  delete+recreate can resolve (eg an invalid combination of fields, or a field enforced
+                  as immutable). By default, such errors are surfaced instead of recreating the resource.
+                type: boolean
               helmChartVersion:
                 description: |-
                   HelmChartVersion indicates the chart version of the Helm release

lib/deployer/applier.go

@@ -28,6 +28,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
@@ -39,6 +40,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	utilyaml "k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/util/retry"
@@ -53,8 +55,57 @@ import (
 
 const (
 	ReasonLabel = "projectsveltos.io/reason"
+
+	recreateDeletePollInterval = 2 * time.Second
+	recreateDeletePollTimeout  = time.Minute
 )
 
+// requiresRecreate returns true if err indicates the API server rejected the request
+// because of an invalid combination of fields (eg a Deployment with strategy.rollingUpdate
+// set while strategy.type is Recreate) or a field enforced as immutable. Either case can
+// only be resolved by deleting and recreating the object; a normal server-side apply patch
+// cannot clear a field it does not own or change a value the server refuses outright.
+func requiresRecreate(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if apierrors.IsInvalid(err) {
+		return true
+	}
+
+	for i := range immutableFieldErrorPatterns {
+		if immutableFieldErrorPatterns[i].MatchString(err.Error()) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// recreateResource deletes object and waits for it to be gone, so a subsequent create
+// does not race against a still-terminating object (eg one with finalizers).
+func recreateResource(ctx context.Context, dr dynamic.ResourceInterface, object *unstructured.Unstructured) error {
+	propagationPolicy := metav1.DeletePropagationBackground
+	err := dr.Delete(ctx, object.GetName(), metav1.DeleteOptions{PropagationPolicy: &propagationPolicy})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return fmt.Errorf("failed to delete %s %s for recreate: %w",
+			object.GetObjectKind().GroupVersionKind().Kind, object.GetName(), err)
+	}
+
+	return wait.PollUntilContextTimeout(ctx, recreateDeletePollInterval, recreateDeletePollTimeout, true,
+		func(ctx context.Context) (bool, error) {
+			_, getErr := dr.Get(ctx, object.GetName(), metav1.GetOptions{})
+			if apierrors.IsNotFound(getErr) {
+				return true, nil
+			}
+			if getErr != nil {
+				return false, getErr
+			}
+			return false, nil
+		})
+}
+
 // CreateNamespace creates a namespace if it does not exist already
 // No action in DryRun mode.
 func CreateNamespace(ctx context.Context, clusterClient client.Client,
@@ -177,6 +228,13 @@ func GetUnstructured(section []byte, logger logr.Logger) ([]*unstructured.Unstru
 var (
 	reCommentLine = regexp.MustCompile(`(?m)^\s*#([^#].*?)$`)
 	reEmptyLine   = regexp.MustCompile(`(?m)^\s*$`)
+
+	// immutableFieldErrorPatterns matches error messages returned by custom admission webhooks
+	// and Kubernetes CEL validation rules that enforce field immutability.
+	immutableFieldErrorPatterns = []*regexp.Regexp{
+		regexp.MustCompile(`.*is\simmutable.*`),
+		regexp.MustCompile(`.*immutable\sfield.*`),
+	}
 )
 
 // removeCommentsAndEmptyLines removes any line containing just YAML comments
@@ -443,7 +501,11 @@ func removeDriftExclusionsFields(ctx context.Context, dr dynamic.ResourceInterfa
 
 // UpdateResource creates or updates a resource in a Cluster.
 // No action in DryRun mode.
-func UpdateResource(ctx context.Context, dr dynamic.ResourceInterface, isDriftDetection, isDryRun bool,
+// When forceRecreate is true, and the apply is rejected with an error that only a
+// delete+recreate can resolve (see requiresRecreate), the object is deleted and recreated
+// instead of returning the error. This never applies to CustomResourceDefinitions, since
+// deleting one cascades to every instance of it.
+func UpdateResource(ctx context.Context, dr dynamic.ResourceInterface, isDriftDetection, isDryRun, forceRecreate bool,
 	driftExclusions []libsveltosv1beta1.DriftExclusion, object *unstructured.Unstructured, subresources []string,
 	logger logr.Logger) (*unstructured.Unstructured, error) {
 
@@ -503,6 +565,14 @@ func UpdateResource(ctx context.Context, dr dynamic.ResourceInterface, isDriftDe
 			}
 			return nil
 		})
+
+		if err != nil && !isDryRun && forceRecreate && requiresRecreate(err) {
+			l.V(logs.LogInfo).Info(fmt.Sprintf("apply rejected (%v), recreating resource", err))
+			if recreateErr := recreateResource(ctx, dr, object); recreateErr != nil {
+				return nil, recreateErr
+			}
+			updatedObject, err = dr.Patch(ctx, object.GetName(), types.ApplyPatchType, data, options)
+		}
 	}
 	if err != nil {
 		return nil, err