diff --git a/cmd/cluster-authentication-operator-tests-ext/main.go b/cmd/cluster-authentication-operator-tests-ext/main.go index fdd363a2ce..be7d997772 100644 --- a/cmd/cluster-authentication-operator-tests-ext/main.go +++ b/cmd/cluster-authentication-operator-tests-ext/main.go @@ -14,6 +14,7 @@ import ( _ "github.com/openshift/cluster-authentication-operator/test/e2e" _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-kms" + _ "github.com/openshift/cluster-authentication-operator/test/e2e-encryption-perf" "k8s.io/klog/v2" ) @@ -92,6 +93,16 @@ func prepareOperatorTestsRegistry() (*oteextension.Registry, error) { ClusterStability: oteextension.ClusterStabilityDisruptive, }) + // ClusterStability set to Disruptive: encryption perf tests trigger API server rollouts. + extension.AddSuite(oteextension.Suite{ + Name: "openshift/cluster-authentication-operator/operator-encryption-perf/serial", + Parallelism: 1, + ClusterStability: oteextension.ClusterStabilityDisruptive, + Qualifiers: []string{ + `name.contains("[Encryption]") && name.contains("[Serial]") && name.contains("Perf")`, + }, + }) + // The following suite runs KMS encryption tests. extension.AddSuite(oteextension.Suite{ Name: "openshift/cluster-authentication-operator/encryption-kms", diff --git a/test/e2e-encryption-perf/encryption_perf.go b/test/e2e-encryption-perf/encryption_perf.go new file mode 100644 index 0000000000..5f4445f7be --- /dev/null +++ b/test/e2e-encryption-perf/encryption_perf.go @@ -0,0 +1,112 @@ +package e2e_encryption_perf + +import ( + "context" + "fmt" + "testing" + "time" + + g "github.com/onsi/ginkgo/v2" + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + configv1 "github.com/openshift/api/config/v1" + oauthapiv1 "github.com/openshift/api/oauth/v1" + operatorv1 "github.com/openshift/api/operator/v1" + oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1" + operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library" + operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + tokenStatsKey = "created oauthaccesstokens" +) + +var _ = g.Describe("[sig-auth] authentication operator", func() { + g.It("[Encryption][Serial] TestPerfEncryptionTypeAESCBC", func(ctx context.Context) { + testPerfEncryptionTypeAESCBC(ctx, g.GinkgoTB()) + }) +}) + +func testPerfEncryptionTypeAESCBC(ctx context.Context, tt testing.TB) { + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + tt.Cleanup(cancel) + clientSet := getPerfClients(tt) + operatorencryption.TestPerfEncryption(ctx, tt, library.PerfScenario{ + BasicScenario: library.BasicScenario{ + Namespace: "openshift-config-managed", + LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", + EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"), + EncryptionConfigSecretNamespace: "openshift-config-managed", + OperatorNamespace: "openshift-authentication-operator", + TargetGRs: operatorencryption.DefaultTargetGRs, + AssertFunc: operatorencryption.AssertTokens, + }, + GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) { + apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, err + } + return apiServerOperator.Status.Conditions, nil + }, + AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) { + require.Empty(t, errorStore, "db loader workers reported errors") + + tokenCount, ok := statStore[tokenStatsKey] + require.True(t, ok, "missing oauth access tokens count stats") + require.GreaterOrEqual(t, tokenCount, 14000) + t.Logf("Created %d access tokens", tokenCount) + }, + AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) { + t.Logf("migration took %v", migrationTime) + expectedMigrationTime := 10 * time.Minute + if migrationTime > expectedMigrationTime { + t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime) + } + }, + DBLoaderWorkers: 3, + DBLoaderFunc: library.DBLoaderRepeat(1, false, + library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)), + EncryptionProvider: library.EncryptionProvider{ + APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionTypeAESCBC}, + }, + }) +} + +func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType { + return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error { + _, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair() + token := &oauthapiv1.OAuthAccessToken{ + ObjectMeta: metav1.ObjectMeta{ + Name: tokenNameHash, + }, + RefreshToken: "I have no special talents. I am only passionately curious", + UserName: "kube:admin", + Scopes: []string{"user:full"}, + RedirectURI: "redirect.me.to.token.of.life", + ClientName: "console", + UserUID: "non-existing-user-id", + } + _, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{}) + return err + } +} + +func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error { + statsCollector(tokenStatsKey) + return nil +} + +func getPerfClients(t testing.TB) operatorencryption.ClientSet { + t.Helper() + + kubeConfig := operatorlibrary.NewClientConfigForTest(t) + + kubeConfig.QPS = 300 + kubeConfig.Burst = 600 + + return operatorencryption.GetClientsFor(t, kubeConfig) +} diff --git a/test/e2e-encryption-perf/encryption_perf_test.go b/test/e2e-encryption-perf/encryption_perf_test.go index 491959931c..4db8d363a0 100644 --- a/test/e2e-encryption-perf/encryption_perf_test.go +++ b/test/e2e-encryption-perf/encryption_perf_test.go @@ -1,109 +1,14 @@ package e2e_encryption_perf import ( - "context" - "errors" - "fmt" "testing" - "time" - - "github.com/stretchr/testify/require" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - - configv1 "github.com/openshift/api/config/v1" - oauthapiv1 "github.com/openshift/api/oauth/v1" - operatorv1 "github.com/openshift/api/operator/v1" - oauthclient "github.com/openshift/client-go/oauth/clientset/versioned/typed/oauth/v1" - operatorlibrary "github.com/openshift/cluster-authentication-operator/test/library" - operatorencryption "github.com/openshift/cluster-authentication-operator/test/library/encryption" - library "github.com/openshift/library-go/test/library/encryption" -) - -const ( - tokenStatsKey = "created oauthaccesstokens" ) +// This test calls the shared test function which +// can be called from both standard Go tests and Ginkgo tests. +// +// This situation is temporary until we verify the new e2e-aws-operator-encryption-perf-serial-ote job. +// Eventually all tests will be run only as part of the OTE framework. func TestPerfEncryptionTypeAESCBC(tt *testing.T) { - ctx := context.TODO() - clientSet := getPerfClients(tt) - library.TestPerfEncryption(tt.Context(), tt, library.PerfScenario{ - BasicScenario: library.BasicScenario{ - Namespace: "openshift-config-managed", - LabelSelector: "encryption.apiserver.operator.openshift.io/component" + "=" + "openshift-oauth-apiserver", - EncryptionConfigSecretName: fmt.Sprintf("encryption-config-%s", "openshift-oauth-apiserver"), - EncryptionConfigSecretNamespace: "openshift-config-managed", - OperatorNamespace: "openshift-authentication-operator", - TargetGRs: operatorencryption.DefaultTargetGRs, - AssertFunc: operatorencryption.AssertTokens, - }, - GetOperatorConditionsFunc: func(t testing.TB) ([]operatorv1.OperatorCondition, error) { - apiServerOperator, err := clientSet.OperatorClient.Get(ctx, "cluster", metav1.GetOptions{}) - if err != nil { - return nil, err - } - return apiServerOperator.Status.Conditions, nil - }, - AssertDBPopulatedFunc: func(t testing.TB, errorStore map[string]int, statStore map[string]int) { - tokenCount, ok := statStore[tokenStatsKey] - if !ok { - err := errors.New("missing oauth access tokens count stats, can't continue the test") - require.NoError(t, err) - } - if tokenCount < 14000 { - err := fmt.Errorf("expected to create at least 14000 tokens but %d were created", tokenCount) - require.NoError(t, err) - } - t.Logf("Created %d access tokens", tokenCount) - }, - AssertMigrationTime: func(t testing.TB, migrationTime time.Duration) { - t.Logf("migration took %v", migrationTime) - expectedMigrationTime := 10 * time.Minute - if migrationTime > expectedMigrationTime { - t.Errorf("migration took too long (%v), expected it to take no more than %v", migrationTime, expectedMigrationTime) - } - }, - DBLoaderWorkers: 3, - DBLoaderFunc: library.DBLoaderRepeat(1, false, - library.DBLoaderRepeatParallel(5010, 50, false, createAccessTokenWrapper(ctx, clientSet.TokenClient), reportSecret)), - EncryptionProvider: library.EncryptionProvider{ - APIServerEncryption: configv1.APIServerEncryption{Type: configv1.EncryptionType("aescbc")}, - }, - }) -} - -func createAccessTokenWrapper(ctx context.Context, tokenClient oauthclient.OAuthAccessTokensGetter) library.DBLoaderFuncType { - return func(_ kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string)) error { - _, tokenNameHash := operatorlibrary.GenerateOAuthTokenPair() - token := &oauthapiv1.OAuthAccessToken{ - ObjectMeta: metav1.ObjectMeta{ - Name: tokenNameHash, - }, - RefreshToken: "I have no special talents. I am only passionately curious", - UserName: "kube:admin", - Scopes: []string{"user:full"}, - RedirectURI: "redirect.me.to.token.of.life", - ClientName: "console", - UserUID: "non-existing-user-id", - } - _, err := tokenClient.OAuthAccessTokens().Create(ctx, token, metav1.CreateOptions{}) - return err - } -} - -func reportSecret(_ kubernetes.Interface, _ string, _ func(error), statsCollector func(string)) error { - statsCollector(tokenStatsKey) - return nil -} - -func getPerfClients(t *testing.T) operatorencryption.ClientSet { - t.Helper() - - kubeConfig := operatorlibrary.NewClientConfigForTest(t) - - kubeConfig.QPS = 300 - kubeConfig.Burst = 600 - - return operatorencryption.GetClientsFor(t, kubeConfig) + testPerfEncryptionTypeAESCBC(tt.Context(), tt) } diff --git a/test/library/encryption/perf_helpers.go b/test/library/encryption/perf_helpers.go new file mode 100644 index 0000000000..4682a571ca --- /dev/null +++ b/test/library/encryption/perf_helpers.go @@ -0,0 +1,185 @@ +package encryption + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + operatorv1 "github.com/openshift/api/operator/v1" + library "github.com/openshift/library-go/test/library/encryption" +) + +const ( + waitPollInterval = 15 * time.Second + waitPollTimeout = 69*time.Minute + 10*time.Minute +) + +// watchForMigrationControllerProgressingConditionAsync starts watching for the migration +// controller progressing condition in a background goroutine. +func watchForMigrationControllerProgressingConditionAsync(ctx context.Context, t testing.TB, getOperatorCondFn library.GetOperatorConditionsFuncType, migrationStartedCh chan time.Time, testStartTime time.Time) { + t.Helper() + go watchForMigrationControllerProgressingCondition(ctx, t, getOperatorCondFn, migrationStartedCh, testStartTime) +} + +// watchForMigrationControllerProgressingCondition waits for the EncryptionMigrationControllerProgressing +// condition to be set to true with a fresh transition timestamp. +// It validates that the condition transition occurred after the test start time to avoid accepting stale conditions. +func watchForMigrationControllerProgressingCondition(ctx context.Context, t testing.TB, getOperatorConditionsFn library.GetOperatorConditionsFuncType, migrationStartedCh chan time.Time, testStartTime time.Time) { + t.Helper() + + t.Logf("Waiting up to %s for the condition %q to be set to true (after %v)", waitPollTimeout.String(), "EncryptionMigrationControllerProgressing", testStartTime) + err := wait.PollUntilContextTimeout(ctx, waitPollInterval, waitPollTimeout, true, func(ctx context.Context) (bool, error) { + conditions, err := getOperatorConditionsFn(t) + if err != nil { + return false, err + } + for _, cond := range conditions { + if cond.Type == "EncryptionMigrationControllerProgressing" && + cond.Status == operatorv1.ConditionTrue && + cond.LastTransitionTime.Time.After(testStartTime) { + t.Logf("EncryptionMigrationControllerProgressing condition observed at %v with reason %q", cond.LastTransitionTime, cond.Reason) + migrationStartedCh <- cond.LastTransitionTime.Time + return true, nil + } + } + return false, nil + }) + if err != nil { + t.Logf("failed waiting for the condition %q to be set to true, err was %v", "EncryptionMigrationControllerProgressing", err) + close(migrationStartedCh) + } +} + +// populateDatabase populates the database using the provided loader function with multiple workers. +func populateDatabase(t testing.TB, workers int, dbLoaderFun library.DBLoaderFuncType, assertDBPopulatedFunc func(t testing.TB, errorStore map[string]int, statStore map[string]int)) { + t.Helper() + start := time.Now() + defer func() { + end := time.Now() + t.Logf("Populating etcd took %v", end.Sub(start)) + }() + + r := newRunner() + + // run executes loaderFunc for each worker + r.run(t, workers, dbLoaderFun) + + assertDBPopulatedFunc(t, r.errorStore, r.statsStore) +} + +// runner manages parallel execution of database loader functions. +type runner struct { + errorStore map[string]int + lock *sync.Mutex + + statsStore map[string]int + lockStats *sync.Mutex + wg *sync.WaitGroup +} + +// newRunner creates a new runner for executing database load functions. +func newRunner() *runner { + r := &runner{} + + r.errorStore = map[string]int{} + r.lock = &sync.Mutex{} + r.statsStore = map[string]int{} + r.lockStats = &sync.Mutex{} + + r.wg = &sync.WaitGroup{} + + return r +} + +// run executes the provided work functions using multiple workers. +func (r *runner) run(t testing.TB, workers int, workFunc ...library.DBLoaderFuncType) { + t.Logf("Executing provided load function for %d workers", workers) + for i := 0; i < workers; i++ { + wrapper := func(wg *sync.WaitGroup) { + defer wg.Done() + kubeClient, err := newKubeClient(t, 300, 600) + if err != nil { + t.Errorf("Unable to create a kube client for a worker due to %v", err) + r.collectError(err) + return + } + if err := runWorkFunctions(kubeClient, "", r.collectError, r.collectStat, workFunc...); err != nil { + t.Logf("worker finished with loader error: %v", err) + } + } + r.wg.Add(1) + go wrapper(r.wg) + } + r.wg.Wait() + if len(r.errorStore) > 0 { + t.Logf("Workers completed with %d distinct error type(s)", len(r.errorStore)) + } else { + t.Log("All workers completed successfully") + } +} + +// collectError collects and counts errors from workers. +func (r *runner) collectError(err error) { + r.lock.Lock() + defer r.lock.Unlock() + errCount, ok := r.errorStore[err.Error()] + if !ok { + r.errorStore[err.Error()] = 1 + return + } + errCount += 1 + r.errorStore[err.Error()] = errCount +} + +// collectStat collects and counts statistics from workers. +func (r *runner) collectStat(stat string) { + r.lockStats.Lock() + defer r.lockStats.Unlock() + statCount, ok := r.statsStore[stat] + if !ok { + r.statsStore[stat] = 1 + return + } + statCount += 1 + r.statsStore[stat] = statCount +} + +// runWorkFunctions executes a series of database loader functions. +func runWorkFunctions(kubeClient kubernetes.Interface, namespace string, errorCollector func(error), statsCollector func(string), workFunc ...library.DBLoaderFuncType) error { + if len(namespace) == 0 { + namespace = createNamespaceName() + } + for _, work := range workFunc { + err := work(kubeClient, namespace, errorCollector, statsCollector) + if err != nil { + errorCollector(err) + return err + } + } + return nil +} + +// createNamespaceName generates a unique namespace name for testing. +func createNamespaceName() string { + return fmt.Sprintf("encryption-%s", rand.String(10)) +} + +// newKubeClient creates a Kubernetes client with specified QPS and burst settings. +func newKubeClient(t testing.TB, qps float32, burst int) (kubernetes.Interface, error) { + kubeConfig := NewClientConfigForTest(t) + + kubeConfig.QPS = qps + kubeConfig.Burst = burst + + kubeClient, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + return kubeClient, nil +} diff --git a/test/library/encryption/scenarios.go b/test/library/encryption/scenarios.go new file mode 100644 index 0000000000..afa980f834 --- /dev/null +++ b/test/library/encryption/scenarios.go @@ -0,0 +1,247 @@ +package encryption + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + + configv1 "github.com/openshift/api/config/v1" + configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" + library "github.com/openshift/library-go/test/library/encryption" +) + +// TestPerfEncryption tests encryption performance. +// This is a local implementation that accepts testing.TB instead of *testing.T. +// It populates the database with test data, enables encryption, and measures migration time. +func TestPerfEncryption(ctx context.Context, tb testing.TB, scenario library.PerfScenario) { + tb.Logf("Starting encryption performance test for %q provider", scenario.EncryptionProvider.Type) + + migrationStartedCh := make(chan time.Time, 1) + + // Create a cancelable context for the watcher goroutine to ensure it stops when the test finishes + watcherCtx, cancel := context.WithCancel(ctx) + tb.Cleanup(cancel) + + // Step 1: Populate the database with test data + tb.Logf("Step 1/3: Populating database with test data using %d workers", scenario.DBLoaderWorkers) + populateDatabase(tb, scenario.DBLoaderWorkers, scenario.DBLoaderFunc, scenario.AssertDBPopulatedFunc) + + // Step 2: Start watching for migration controller progressing condition asynchronously + // Capture test start time to validate fresh condition transitions + testStartTime := time.Now() + tb.Logf("Step 2/3: Starting migration progress monitor (test start time: %v)", testStartTime) + watchForMigrationControllerProgressingConditionAsync(watcherCtx, tb, scenario.GetOperatorConditionsFunc, migrationStartedCh, testStartTime) + + // Step 3: Run encryption test and measure time + tb.Logf("Step 3/3: Enabling encryption and measuring migration time") + endTimeStamp := runTestEncryptionPerf(tb, scenario) + + // Calculate and assert migration time + select { + case migrationStarted := <-migrationStartedCh: + if migrationStarted.IsZero() { + tb.Error("unable to calculate the migration time, migration watcher encountered an error") + } else { + migrationTime := endTimeStamp.Sub(migrationStarted) + tb.Logf("Migration completed in %v", migrationTime) + scenario.AssertMigrationTime(tb, migrationTime) + } + case <-time.After(waitPollTimeout): + tb.Error("unable to calculate the migration time, failed to observe when the migration has started") + } + + tb.Logf("Encryption performance test completed") +} + +func runTestEncryptionPerf(tb testing.TB, scenario library.PerfScenario) time.Time { + var ts time.Time + TestEncryptionType(tb, library.BasicScenario{ + Namespace: scenario.Namespace, + LabelSelector: scenario.LabelSelector, + EncryptionConfigSecretName: scenario.EncryptionConfigSecretName, + EncryptionConfigSecretNamespace: scenario.EncryptionConfigSecretNamespace, + OperatorNamespace: scenario.OperatorNamespace, + TargetGRs: scenario.TargetGRs, + AssertFunc: func(t testing.TB, clientSet library.ClientSet, expectedMode configv1.EncryptionType, namespace, labelSelector string) { + // Note that AssertFunc is executed after an encryption secret has been annotated + ts = time.Now() + scenario.AssertFunc(t, clientSet, expectedMode, scenario.Namespace, scenario.LabelSelector) + tb.Logf("AssertFunc for TestEncryption scenario with %q provider took %v", scenario.EncryptionProvider.Type, time.Since(ts)) + }, + }, scenario.EncryptionProvider) + return ts +} + +// TestEncryptionType is a helper that dispatches to the appropriate encryption type test. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionType(tb testing.TB, scenario library.BasicScenario, provider library.EncryptionProvider) { + switch provider.Type { + case configv1.EncryptionTypeAESCBC: + TestEncryptionTypeAESCBC(tb, scenario) + case configv1.EncryptionTypeAESGCM: + TestEncryptionTypeAESGCM(tb, scenario) + case configv1.EncryptionTypeKMS: + TestEncryptionTypeKMS(tb, scenario) + case configv1.EncryptionTypeIdentity: + TestEncryptionTypeIdentity(tb, scenario) + case "": + TestEncryptionTypeUnset(tb, scenario) + default: + tb.Fatalf("Unknown encryption type: %s", provider.Type) + } +} + +// TestEncryptionTypeIdentity tests encryption with identity mode (no encryption). +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeIdentity(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeIdentity, configv1.EncryptionTypeIdentity) +} + +// TestEncryptionTypeUnset tests encryption with unset mode (defaults to identity). +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeUnset(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, "", configv1.EncryptionTypeIdentity) +} + +// TestEncryptionTypeAESCBC tests encryption with AESCBC mode. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeAESCBC(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeAESCBC, configv1.EncryptionTypeAESCBC) +} + +// TestEncryptionTypeAESGCM tests encryption with AESGCM mode. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeAESGCM(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeAESGCM, configv1.EncryptionTypeAESGCM) +} + +// TestEncryptionTypeKMS tests KMS encryption. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func TestEncryptionTypeKMS(tb testing.TB, scenario library.BasicScenario) { + testEncryptionTypeBase(tb, scenario, configv1.EncryptionTypeKMS, configv1.EncryptionTypeKMS) +} + +// testEncryptionTypeBase is the base implementation for all encryption type tests. +func testEncryptionTypeBase(tb testing.TB, scenario library.BasicScenario, encryptionType configv1.EncryptionType, expectedType configv1.EncryptionType) { + if encryptionType == "" { + tb.Logf("Starting encryption e2e test for unset mode (defaults to identity)") + } else { + tb.Logf("Starting encryption e2e test for %q mode", encryptionType) + } + + clientSet := SetAndWaitForEncryptionType(tb, encryptionType, scenario.TargetGRs, scenario.Namespace, scenario.LabelSelector) + + scenario.AssertFunc(tb, clientSet, expectedType, scenario.Namespace, scenario.LabelSelector) + + // For actual encryption types (not identity/unset), also assert encryption config + if encryptionType != "" && encryptionType != configv1.EncryptionTypeIdentity { + library.AssertEncryptionConfig(tb, clientSet, scenario.EncryptionConfigSecretName, scenario.EncryptionConfigSecretNamespace, scenario.TargetGRs) + } +} + +// createLibraryClientSet creates a library.ClientSet from kubeconfig. +// This helper consolidates the duplicated clientset creation logic. +func createLibraryClientSet(tb testing.TB) library.ClientSet { + kubeConfig := NewClientConfigForTest(tb) + libClientSet := library.ClientSet{} + libClientSet.Kube = kubernetes.NewForConfigOrDie(kubeConfig) + libClientSet.Etcd = library.NewEtcdClient(libClientSet.Kube) + configClient := configv1client.NewForConfigOrDie(kubeConfig) + libClientSet.ApiServerConfig = configClient.APIServers() + return libClientSet +} + +// TestEncryptionTurnOnAndOff tests turning encryption on and off. +// This is a local implementation that accepts testing.TB instead of *testing.T. +// It runs through a complete cycle twice to ensure repeatability: +// 1. Create resource -> Enable encryption -> Verify encrypted -> Disable -> Verify not encrypted +// 2. Repeat the cycle to ensure it works multiple times +func TestEncryptionTurnOnAndOff(tb testing.TB, scenario library.OnOffScenario) { + tb.Logf("Starting encryption turn-on-and-off test for resource %q", scenario.ResourceName) + + // Helper to get library clientset - uses shared helper function + getLibClientSet := func() library.ClientSet { + return createLibraryClientSet(tb) + } + + // Step 1: Create and store the resource + tb.Logf("Step 1/9: Creating and storing %s", scenario.ResourceName) + scenario.CreateResourceFunc(tb, getLibClientSet(), scenario.Namespace) + + // Step 2: Turn on encryption with the specified provider + tb.Logf("Step 2/9: Enabling %s encryption", scenario.EncryptionProvider.Type) + TestEncryptionType(tb, scenario.BasicScenario, scenario.EncryptionProvider) + + // Step 3: Assert the resource is encrypted + tb.Logf("Step 3/9: Verifying %s is encrypted", scenario.ResourceName) + scenario.AssertResourceEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 4: Turn off encryption (switch to identity mode) + tb.Logf("Step 4/9: Disabling encryption (switching to identity mode)") + TestEncryptionTypeIdentity(tb, scenario.BasicScenario) + + // Step 5: Assert the resource is not encrypted + tb.Logf("Step 5/9: Verifying %s is not encrypted", scenario.ResourceName) + scenario.AssertResourceNotEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 6: Turn on encryption again (second cycle to test repeatability) + tb.Logf("Step 6/9: Enabling %s encryption (second cycle)", scenario.EncryptionProvider.Type) + TestEncryptionType(tb, scenario.BasicScenario, scenario.EncryptionProvider) + + // Step 7: Assert the resource is encrypted again + tb.Logf("Step 7/9: Verifying %s is encrypted (second cycle)", scenario.ResourceName) + scenario.AssertResourceEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + // Step 8: Turn off encryption again (second cycle) + tb.Logf("Step 8/9: Disabling encryption (identity mode, second cycle)") + TestEncryptionTypeIdentity(tb, scenario.BasicScenario) + + // Step 9: Assert the resource is not encrypted again + tb.Logf("Step 9/9: Verifying %s is not encrypted (second cycle)", scenario.ResourceName) + scenario.AssertResourceNotEncryptedFunc(tb, getLibClientSet(), scenario.ResourceFunc(tb, scenario.Namespace)) + + tb.Logf("Encryption turn-on-and-off test completed successfully") +} + +// SetAndWaitForEncryptionType sets the encryption type and waits for encryption to complete. +// This is a local implementation that accepts testing.TB instead of *testing.T. +func SetAndWaitForEncryptionType(tb testing.TB, encryptionType configv1.EncryptionType, defaultTargetGRs []schema.GroupResource, namespace, labelSelector string) library.ClientSet { + // Create library clientset using shared helper + libClientSet := createLibraryClientSet(tb) + + lastMigratedKeyMeta, err := library.GetLastKeyMeta(tb, libClientSet.Kube, namespace, labelSelector) + require.NoError(tb, err) + + // Update encryption type with retry on conflict + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + reqCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Get current API server config + apiServer, err := libClientSet.ApiServerConfig.Get(reqCtx, "cluster", metav1.GetOptions{}) + if err != nil { + return err + } + + // Update encryption type if needed + if apiServer.Spec.Encryption.Type != encryptionType { + tb.Logf("Updating encryption type in the config file for APIServer to %q", encryptionType) + apiServer.Spec.Encryption.Type = encryptionType + _, err = libClientSet.ApiServerConfig.Update(reqCtx, apiServer, metav1.UpdateOptions{}) + return err + } + tb.Logf("APIServer is already configured to use %q mode", encryptionType) + return nil + }) + require.NoError(tb, err) + + library.WaitForEncryptionKeyBasedOn(tb, libClientSet.Kube, lastMigratedKeyMeta, encryptionType, defaultTargetGRs, namespace, labelSelector) + + return libClientSet +}