From 1c2e1a5c84bfc5210dc9347dafb261a17a6c6b0c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Dec 2023 14:13:46 +0100 Subject: [PATCH 1/8] CKS: retry if unable to drain node or unable to upgrade k8s node I tried CKS upgrade 16 times, 11 of 16 upgrades succeeded. 2 of 16 upgrades failed due to ``` error: unable to drain node "testcluster-of7974-node-18c8c33c2c3" due to error:[error when evicting pods/"cloud-controller-manager-5b8fc87665-5nwlh" -n "kube-system": Post "https://10.0.66.18:6443/api/v1/namespaces/kube-system/pods/cloud-controller-manager-5b8fc87665-5nwlh/eviction": unexpected EOF, error when evicting pods/"coredns-5d78c9869d-h5nkz" -n "kube-system": Post "https://10.0.66.18:6443/api/v1/namespaces/kube-system/pods/coredns-5d78c9869d-h5nkz/eviction": unexpected EOF], continuing command... ``` 3 of 16 upgrades failed due to ``` Error from server: error when retrieving current configuration of: Resource: "rbac.authorization.k8s.io/v1, Resource=roles", GroupVersionKind: "rbac.authorization.k8s.io/v1, Kind=Role" Name: "kubernetes-dashboard", Namespace: "kubernetes-dashboard" from server for: "/mnt/k8sdisk//dashboard.yaml": etcdserver: leader changed ``` --- .../cluster/KubernetesClusterManagerImpl.java | 1 + .../cluster/KubernetesClusterService.java | 6 ++ .../KubernetesClusterUpgradeWorker.java | 61 +++++++++++++------ 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 19c66c1ba0b8..4bd6ec65fc3a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1647,6 +1647,7 @@ public ConfigKey[] getConfigKeys() { KubernetesClusterStartTimeout, KubernetesClusterScaleTimeout, KubernetesClusterUpgradeTimeout, + KubernetesClusterUpgradeRetries, KubernetesClusterExperimentalFeaturesEnabled, KubernetesMaxClusterSize }; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 420f35527207..12240a47ecac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -65,6 +65,12 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Timeout interval (in seconds) in which upgrade operation for a Kubernetes cluster should be completed. Not strictly obeyed while upgrade is in progress on a node", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterUpgradeRetries = new ConfigKey("Advanced", Integer.class, + "cloud.kubernetes.cluster.upgrade.retries", + "3", + "The number of retries if fail to upgrade kubernetes cluster due to some reasons (e.g. drain node, etcdserver leader changed)", + true, + KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.cluster.experimental.features.enabled", "false", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 14f5760d5ae6..ee27ed93205d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -77,39 +77,62 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in } private void upgradeKubernetesClusterNodes() { - Pair result = null; for (int i = 0; i < clusterVMs.size(); ++i) { UserVm vm = clusterVMs.get(i); String hostName = vm.getHostName(); if (StringUtils.isNotEmpty(hostName)) { hostName = hostName.toLowerCase(); } - result = null; + Pair result; if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Upgrading node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s", vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } - try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), - 10000, 10000, 60000); - } catch (Exception e) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); - } - if (!result.first()) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { + String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); + try { + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, + String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), + 10000, 10000, 60000); + if (result.first()) { + break; + } + if (retry > 0) { + LOGGER.error(String.format("%s, retries left: %s", errorMessage, retry)); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + } catch (Exception e) { + if (retry > 0) { + LOGGER.error(String.format("%s due to %s, retries left: %s", errorMessage, e, retry)); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); + } + } } if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } - try { - deployProvider(); - result = runInstallScriptOnVM(vm, i); - } catch (Exception e) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); - } - if (!result.first()) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { + String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); + try { + deployProvider(); + result = runInstallScriptOnVM(vm, i); + if (result.first()) { + break; + } + if (retry > 0) { + LOGGER.error(String.format("%s, retries left: %s", errorMessage, retry)); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + } catch (Exception e) { + if (retry > 0) { + LOGGER.error(String.format("%s due to %s, retries left: %s", errorMessage, e, retry)); + } else { + logTransitStateDetachIsoAndThrow(Level.ERROR, errorMessage, kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); + } + } } if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); From 62d6733cd6675b6a81d9965e6f3443a227d5e835 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Dec 2023 14:04:15 +0100 Subject: [PATCH 2/8] CKS: remove tests of creating/deleting HA clusters as they are covered by the upgrade test --- .../smoke/test_kubernetes_clusters.py | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index dc8d42f444ca..bf15f1675784 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -523,24 +523,6 @@ def test_06_delete_kubernetes_cluster(self): return - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_07_deploy_kubernetes_ha_cluster(self): - """Test to deploy a new HA Kubernetes cluster - - # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3) - self.debug("HA Kubernetes cluster with ID: %s successfully deployed" % k8s_cluster.id) - return - @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_08_upgrade_kubernetes_ha_cluster(self): @@ -568,24 +550,6 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_09_delete_kubernetes_ha_cluster(self): - """Test to delete a HA Kubernetes cluster - - # Validate the following: - # 1. deleteKubernetesCluster should delete an existing HA Kubernetes cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3) - - self.debug("Deleting Kubernetes cluster with ID: %s" % k8s_cluster.id) - return - @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_10_vpc_tier_kubernetes_cluster(self): From af93915634b2cdfbbb199737a0b0e75dd50fc7bd Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Dec 2023 14:11:45 +0100 Subject: [PATCH 3/8] CKS: run HA cluster upgrade test 20 times ...will revert later it will take 15m * 20 = 5 hours --- .../smoke/test_kubernetes_clusters.py | 486 ++++++++++++++++++ 1 file changed, 486 insertions(+) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index bf15f1675784..86935937f6fb 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -550,6 +550,492 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_2_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_3_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_4_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_5_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_6_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_7_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_8_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_9_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_10_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_11_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_12_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_13_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_14_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_15_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_16_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_17_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_18_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_19_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_10_vpc_tier_kubernetes_cluster(self): From cc3e8c00b06e574f4299c1aab06fa314f5e06deb Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Sun, 24 Dec 2023 09:54:55 +0100 Subject: [PATCH 4/8] Revert "CKS: run HA cluster upgrade test 20 times ...will revert later" This reverts commit af93915634b2cdfbbb199737a0b0e75dd50fc7bd. --- .../smoke/test_kubernetes_clusters.py | 486 ------------------ 1 file changed, 486 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 86935937f6fb..bf15f1675784 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -550,492 +550,6 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_2_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_3_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_4_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_5_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_6_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_7_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_8_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_9_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_10_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_11_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_12_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_13_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_14_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_15_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_16_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_17_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_18_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_19_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_10_vpc_tier_kubernetes_cluster(self): From e3cc9de41eacd015ad2ba9c022d97b8474e451b4 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 25 Jan 2024 09:33:22 +0100 Subject: [PATCH 5/8] Update PR 8402 as suggested --- .../cluster/actionworkers/KubernetesClusterUpgradeWorker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ee27ed93205d..d418e20f58f7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -88,8 +88,8 @@ private void upgradeKubernetesClusterNodes() { LOGGER.info(String.format("Upgrading node on VM %s in Kubernetes cluster %s with Kubernetes version(%s) ID: %s", vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } + String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { - String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to drain Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); try { result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-emptydir-data", hostName), @@ -113,8 +113,8 @@ private void upgradeKubernetesClusterNodes() { if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } + errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); for (int retry = KubernetesClusterService.KubernetesClusterUpgradeRetries.value(); retry >= 0; retry--) { - String errorMessage = String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()); try { deployProvider(); result = runInstallScriptOnVM(vm, i); From 34e227ada4fab0f66d47570d43de15a46a429007 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 22 Dec 2023 14:11:45 +0100 Subject: [PATCH 6/8] CKS: run HA cluster upgrade test 20 times ...will revert later it will take 15m * 20 = 5 hours --- .../smoke/test_kubernetes_clusters.py | 486 ++++++++++++++++++ 1 file changed, 486 insertions(+) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index bf15f1675784..86935937f6fb 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -550,6 +550,492 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_2_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_3_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_4_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_5_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_6_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_7_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_8_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_9_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_10_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_11_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_12_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_13_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_14_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_15_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_16_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_17_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_18_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorNotSupported") + def test_08_19_upgrade_kubernetes_ha_cluster(self): + """Test to upgrade a HA Kubernetes cluster to newer version + + # Validate the following: + # 1. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + if self.default_network: + self.skipTest("HA cluster on shared network requires external ip address, skipping it") + global k8s_cluster + k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) + time.sleep(self.services["sleep"]) + + self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) + try: + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) + except Exception as e: + self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) + + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) + return + @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_10_vpc_tier_kubernetes_cluster(self): From 1d20dfe7a133ccca07c0d037d35f283b731de8d5 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 26 Jan 2024 13:27:54 +0100 Subject: [PATCH 7/8] test: remove CKS cluster if fail to create or verify --- test/integration/smoke/test_kubernetes_clusters.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 86935937f6fb..2437d9ba3e68 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -278,13 +278,15 @@ def deleteKubernetesSupportedVersion(cls, version_id): cls.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) @classmethod - def listKubernetesCluster(cls, cluster_id = None): + def listKubernetesCluster(cls, cluster_id = None, cluster_name = None): listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() listKubernetesClustersCmd.listall = True if cluster_id != None: listKubernetesClustersCmd.id = cluster_id + if cluster_name != None: + listKubernetesClustersCmd.name = cluster_name clusterResponse = cls.apiclient.listKubernetesClusters(listKubernetesClustersCmd) - if cluster_id != None and clusterResponse != None: + if (cluster_id != None or cluster_name != None) and clusterResponse != None: return clusterResponse[0] return clusterResponse @@ -1189,8 +1191,14 @@ def createNewKubernetesCluster(self, version, size, control_nodes) : cluster = self.createKubernetesCluster(name, version.id, size, control_nodes) self.verifyKubernetesCluster(cluster, name, version.id, size, control_nodes) except Exception as ex: + cluster = self.listKubernetesCluster(cluster_name = name) + if cluster != None: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) self.fail("Kubernetes cluster deployment failed: %s" % ex) except AssertionError as err: + cluster = self.listKubernetesCluster(cluster_name = name) + if cluster != None: + self.deleteKubernetesClusterAndVerify(cluster.id, False, True) self.fail("Kubernetes cluster deployment failed during cluster verification: %s" % err) return cluster From 232d52c6910f84da91cd892441b46e007e3e51ca Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 6 Feb 2024 11:12:56 +0100 Subject: [PATCH 8/8] Revert "CKS: run HA cluster upgrade test 20 times ...will revert later" This reverts commit 34e227ada4fab0f66d47570d43de15a46a429007. --- .../smoke/test_kubernetes_clusters.py | 486 ------------------ 1 file changed, 486 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 2437d9ba3e68..0c5b96a77122 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -552,492 +552,6 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_2_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_3_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_4_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_5_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_6_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_7_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_8_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_9_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_10_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_11_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_12_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_13_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_14_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_15_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_16_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_17_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_18_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - @skipTestIf("hypervisorNotSupported") - def test_08_19_upgrade_kubernetes_ha_cluster(self): - """Test to upgrade a HA Kubernetes cluster to newer version - - # Validate the following: - # 1. upgradeKubernetesCluster should return valid info for the cluster - """ - if self.setup_failed == True: - self.fail("Setup incomplete") - if self.default_network: - self.skipTest("HA cluster on shared network requires external ip address, skipping it") - global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(1, 3, version=self.kubernetes_version_v1) - time.sleep(self.services["sleep"]) - - self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) - try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_v2.id) - except Exception as e: - self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_v2.id) - self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) - return - @attr(tags=["advanced", "smoke"], required_hardware="true") @skipTestIf("hypervisorNotSupported") def test_10_vpc_tier_kubernetes_cluster(self):