From 1c0b1238b9c7d1ca261bb3eb928dbb19a5933d00 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 14 Jan 2022 11:41:54 +0530 Subject: [PATCH 01/22] This PR/commit comprises of the following: - Support to fallback on the older systemVM template in case of no change in template across ACS versions - Update core user to cloud in CKS - Display details of accessing CKS nodes in the UI - K8s Access tab - Update systemvm template from debian 11 to debian 11.2 - Update letsencrypt cert - Remove docker dependency as from ACS 4.16 onward k8s has deprecated support for docker - use containerd as container runtime --- .gitignore | 1 + .../main/java/com/cloud/agent/AgentShell.java | 50 +++++++++---------- engine/schema/pom.xml | 8 +-- .../cloud/upgrade/DatabaseUpgradeChecker.java | 9 ++-- .../upgrade/SystemVmTemplateRegistration.java | 22 ++------ .../META-INF/db/schema-41600to41610.sql | 1 + engine/schema/templateConfig.sh | 20 +++++--- .../KubernetesClusterActionWorker.java | 42 ++++++++-------- .../resources/conf/k8s-control-node-add.yml | 21 ++++---- .../main/resources/conf/k8s-control-node.yml | 23 +++++---- .../src/main/resources/conf/k8s-node.yml | 21 ++++---- .../resources/script/upgrade-kubernetes.sh | 2 +- pom.xml | 1 + .../util/create-kubernetes-binaries-iso.sh | 22 ++++---- .../debian/opt/cloud/bin/setup/cksnode.sh | 2 +- .../scripts/configure_systemvm_services.sh | 13 +++-- .../scripts/install_systemvm_packages.sh | 2 +- .../appliance/systemvmtemplate/template.json | 4 +- ui/public/locales/en.json | 2 + ui/src/views/compute/KubernetesServiceTab.vue | 3 ++ .../java/com/cloud/utils/EncryptionUtil.java | 30 ++++++++--- .../java/com/cloud/utils/script/Script.java | 19 ++++--- .../com/cloud/utils/ssh/SSHCmdHelper.java | 15 +++--- 23 files changed, 173 insertions(+), 160 deletions(-) diff --git a/.gitignore b/.gitignore index b67dc8e6499f..011a4b005dce 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ tools/cli/cloudmonkey/precache.py tools/marvin/marvin/cloudstackAPI/ tools/marvin/build/ tools/cli/build/ +tools/appliance/systemvmtemplate/packer_cache/ *.jar *.war *.mar diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index 01654ac9caa9..0ab5b7863a85 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -16,6 +16,26 @@ // under the License. package com.cloud.agent; +import com.cloud.agent.Agent.ExitStatus; +import com.cloud.agent.dao.StorageComponent; +import com.cloud.agent.dao.impl.PropertiesStorage; +import com.cloud.resource.ServerResource; +import com.cloud.utils.LogUtils; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.ProcessUtil; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.backoff.BackoffAlgorithm; +import com.cloud.utils.backoff.impl.ConstantTimeBackoff; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Strings; +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; +import org.apache.commons.daemon.DaemonInitException; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.DOMConfigurator; + +import javax.naming.ConfigurationException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,28 +50,6 @@ import java.util.Properties; import java.util.UUID; -import javax.naming.ConfigurationException; - -import org.apache.commons.daemon.Daemon; -import org.apache.commons.daemon.DaemonContext; -import org.apache.commons.daemon.DaemonInitException; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.xml.DOMConfigurator; - -import com.cloud.agent.Agent.ExitStatus; -import com.cloud.agent.dao.StorageComponent; -import com.cloud.agent.dao.impl.PropertiesStorage; -import com.cloud.resource.ServerResource; -import com.cloud.utils.LogUtils; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.ProcessUtil; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.backoff.BackoffAlgorithm; -import com.cloud.utils.backoff.impl.ConstantTimeBackoff; -import com.cloud.utils.exception.CloudRuntimeException; -import com.google.common.base.Strings; - public class AgentShell implements IAgentShell, Daemon { private static final Logger s_logger = Logger.getLogger(AgentShell.class.getName()); @@ -423,13 +421,13 @@ private void launchAgentFromClassInfo(String resourceClassNames) throws Configur } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { - throw new ConfigurationException("Security excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Security exception when loading resource: " + name + " due to: " + e.toString()); } catch (final NoSuchMethodException e) { - throw new ConfigurationException("Method not found excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Method not found exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalArgumentException e) { - throw new ConfigurationException("Illegal argument excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Illegal argument exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InstantiationException e) { - throw new ConfigurationException("Instantiation excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Instantiation exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalAccessException e) { throw new ConfigurationException("Illegal access exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InvocationTargetException e) { diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 867535251547..588ed92a0d87 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -73,10 +73,10 @@ - def projectVersion = project.version + def projectVersion = project.properties['project.systemvm.template.version'] String[] versionParts = projectVersion.tokenize('.') - pom.properties['cs.version'] = "4.16" - pom.properties['patch.version'] = "0" + pom.properties['cs.version'] = versionParts[0] + "." + versionParts[1] + pom.properties['patch.version'] = versionParts[2] @@ -146,7 +146,7 @@ bash templateConfig.sh - ${project.version} + ${project.systemvm.template.version} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index cf3f7282f9f6..3c6abbc6e86d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -367,10 +367,11 @@ public void check() { return; } - SystemVmTemplateRegistration.parseMetadataFile(); - final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(currentVersion.getMajorRelease()) + "." + String.valueOf(currentVersion.getMinorRelease()); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(currentVersion.getPatchRelease()); + String csVersion = SystemVmTemplateRegistration.parseMetadataFile(); + final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); + final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); s_logger.info("DB version = " + dbVersion + " Code Version = " + currentVersion); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index d788adac8244..4119b11ae508 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,6 +36,7 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -54,7 +55,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -64,7 +64,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -351,16 +350,6 @@ public static boolean validateIfSeeded(String url, String path) { } } - private String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - LOGGER.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - public Long getRegisteredTemplateId(Pair hypervisorAndTemplateName) { VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second()); Long templateId = null; @@ -690,7 +679,7 @@ public void registerTemplate(Pair hypervisorA } } - public static void parseMetadataFile() { + public static String parseMetadataFile() { try { Ini ini = new Ini(); ini.load(new FileReader(METADATA_FILE)); @@ -702,6 +691,8 @@ public static void parseMetadataFile() { NewTemplateChecksum.put(hypervisorType, section.get("checksum")); NewTemplateUrl.put(hypervisorType, section.get("downloadurl")); } + Ini.Section section = ini.get("default"); + return section.get("version"); } catch (Exception e) { String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); LOGGER.error(errMsg, e); @@ -735,7 +726,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = calculateChecksum(tempFile); + String templateChecksum = EncryptionUtil.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; @@ -812,9 +803,6 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index 04bdbca56b11..7c23f1178226 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -20,3 +20,4 @@ --; ALTER TABLE `cloud`.`vm_work_job` ADD COLUMN `secondary_object` char(100) COMMENT 'any additional item that must be checked during queueing' AFTER `vm_instance_id`; +UPDATE `cloud`.`vm_template` set deploy_as_is = 0 where id = 8; \ No newline at end of file diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index c3093530e7b2..891c73db1bbf 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -23,8 +23,10 @@ function getTemplateVersion() { subversion1="$(cut -d'.' -f1 <<<"$version")" subversion2="$(cut -d'.' -f2 <<<"$version")" minorversion="$(cut -d'.' -f3 <<<"$version")" + securityversion="$(cut -d'.' -f4 <<<"$version")" export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" + export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" } function getGenericName() { @@ -52,12 +54,14 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) + echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE for i in "${!templates[@]}" do section="$i" hvName=$(getGenericName $i) - templatename="systemvm-${i}-${CS_VERSION}.${CS_MINOR_VERSION}" - checksum=$(getChecksum "$fileData" $hvName) + + templatename="systemvm-${i}-${VERSION}" + checksum=$(getChecksum "$fileData" "$VERSION-$hvName") downloadurl="${templates[$i]}" filename=$(echo ${downloadurl##*'/'}) echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\n" >> $METADATAFILE @@ -66,12 +70,12 @@ function createMetadataFile() { declare -A templates getTemplateVersion $1 -templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-kvm.qcow2.bz2" - ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-vmware.ova" - ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-xen.vhd.bz2" - ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-hyperv.vhd.zip" - ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-kvm.qcow2.bz2" - ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-ovm.raw.bz2" ) +templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-vmware.ova" + ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-xen.vhd.bz2" + ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-hyperv.vhd.zip" + ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-ovm.raw.bz2" ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 62d45a3e028d..cf0403b4000a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -17,27 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.ca.CAManager; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.VlanDao; @@ -82,10 +61,29 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; public class KubernetesClusterActionWorker { - public static final String CLUSTER_NODE_VM_USER = "core"; + public static final String CLUSTER_NODE_VM_USER = "cloud"; public static final int CLUSTER_API_PORT = 6443; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 0891d2015b30..db295188eba5 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +192,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +210,16 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,6 +241,7 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 322f08099cf7..730477bd4eef 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -51,7 +51,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -138,7 +138,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -187,7 +187,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -198,7 +198,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -216,7 +216,7 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -231,7 +231,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -267,16 +267,16 @@ write_files: /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -298,6 +298,7 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 28ba43e75845..eea369921ac8 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +192,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +210,16 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,5 +241,6 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 7e0c3c00a8a5..b85ea00e9ed6 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" done <<< "$output" fi if [ -e "${BINARIES_DIR}/provider.yaml" ]; then diff --git a/pom.xml b/pom.xml index 67cbc6e2e5bd..6813bb867c3e 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ UTF-8 UTF-8 + 4.16.1.0 11 diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index ba3dca77a8d2..aec95241eb96 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -98,19 +98,18 @@ provider_conf_file="${working_dir}/provider.yaml" curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} echo "Fetching k8s docker images..." -docker -v +ctr -v if [ $? -ne 0 ]; then - echo "Installing docker..." + echo "Installing containerd..." if [ -f /etc/redhat-release ]; then sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm - sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce - sudo systemctl enable docker && sudo systemctl start docker + sudo yum install -y containerd.io elif [ -f /etc/lsb-release ]; then - sudo apt update && sudo apt install docker.io -y - sudo systemctl enable docker && sudo systemctl start docker + sudo apt update && sudo apt install containerd.io -y fi + sudo systemctl enable containerd && sudo systemctl start containerd fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` @@ -130,11 +129,14 @@ provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' output=`printf "%s\n" ${output} ${provider_image}` while read -r line; do - echo "Downloading docker image $line ---" - sudo docker pull "$line" + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` - sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" - sudo docker image rm "$line" + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index a864d188d009..240e15f95ff2 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -63,7 +63,7 @@ setup_k8s_node() { systemctl enable --now --no-block containerd systemctl enable --now --no-block docker.socket systemctl enable --now --no-block docker.service - if [ -f /home/core/success ]; then + if [ -f /home/cloud/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final else diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index db3eec5cb40c..8c844b893b53 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -50,9 +50,12 @@ function configure_cacerts() { CDIR=$(pwd) cd /tmp # Add LetsEncrypt ca-cert - wget https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der - keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx3cross -file lets-encrypt-x3-cross-signed.der - rm -f lets-encrypt-x3-cross-signed.der + wget https://letsencrypt.org/certs/lets-encrypt-r3.der + wget https://letsencrypt.org/certs/isrgrootx1.der + + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityr3 -file lets-encrypt-r3.der + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx1 -file isrgrootx1.der + rm -f lets-encrypt-r3.der isrgrootx1.der cd $CDIR } @@ -126,10 +129,6 @@ function configure_services() { # Disable container services systemctl disable containerd - systemctl disable docker.service - systemctl stop docker.service - systemctl disable docker.socket - systemctl stop docker.socket # Disable cloud init by default cat < /etc/cloud/cloud.cfg.d/cloudstack.cfg diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 555a00c28edb..3b4ef6537286 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -98,7 +98,7 @@ function install_packages() { apt-key fingerprint 0EBFCD88 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" apt-get update - ${apt_get} install docker-ce docker-ce-cli containerd.io + ${apt_get} install containerd.io apt_clean diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index bd932bf6cdef..31eb2ea5abf1 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:5f6aed67b159d7ccc1a90df33cc8a314aa278728a6f50707ebf10c02e46664e383ca5fa19163b0a1c6a4cb77a39587881584b00b45f512b4a470f1138eaa1801", - "iso_url": "https://cdimage.debian.org/debian-cd/11.0.0/amd64/iso-cd/debian-11.0.0-amd64-netinst.iso", + "iso_checksum": "sha512:c685b85cf9f248633ba3cd2b9f9e781fa03225587e0c332aef2063f6877a1f0622f56d44cf0690087b0ca36883147ecb5593e3da6f965968402cdbdf12f6dd74", + "iso_url": "https://cdimage.debian.org/debian-cd/11.2.0/amd64/iso-cd/debian-11.2.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index ad0173452211..9137e0cb9f86 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,7 @@ "label.accept.project.invitation": "Accept project invitation", "label.access": "Access", "label.accesskey": "Access Key", +"label.acess.kubernetes.nodes": "Access Kubernetes Nodes", "label.account": "Account", "label.account.and.security.group": "Account - Security group", "label.account.details": "Account details", @@ -1232,6 +1233,7 @@ "label.keypair": "SSH Key Pair", "label.kubeconfig.cluster": "Kubernetes Cluster Config", "label.kubernetes": "Kubernetes", +"label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using:
ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address]

where,
ssh_key: points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server.
port_number: can be obtained from the Port Forwarding Tab (Public Port column)", "label.kubernetes.cluster": "Kubernetes cluster", "label.kubernetes.cluster.create": "Create Kubernetes Cluster", "label.kubernetes.cluster.delete": "Delete Kubernetes Cluster", diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 97f1c5456ece..039d09eab87e 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -88,6 +88,9 @@

{{ $t('label.more.access.dashboard.ui') }}, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui

+ +

+ { private static final Logger s_logger = Logger.getLogger(Script.class); @@ -320,7 +319,7 @@ public String call() { try { _logger.trace("Checking exit value of process"); _process.exitValue(); - _logger.trace("Script ran within the alloted time"); + _logger.trace("Script ran within the allotted time"); } catch (IllegalThreadStateException e) { _logger.warn("Interrupting script."); _isTimeOut = true; diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 291f8c1b43fd..dc4333be6a1a 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -19,16 +19,15 @@ package com.cloud.utils.ssh; -import java.io.IOException; -import java.io.InputStream; - +import com.google.common.base.Strings; +import com.trilead.ssh2.ChannelCondition; +import com.trilead.ssh2.Session; import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; -import com.google.common.base.Strings; -import com.trilead.ssh2.ChannelCondition; -import com.trilead.ssh2.Session; +import java.io.IOException; +import java.io.InputStream; public class SSHCmdHelper { private static final Logger s_logger = Logger.getLogger(SSHCmdHelper.class); @@ -242,8 +241,8 @@ public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshC } return result; } catch (Exception e) { - s_logger.debug("Ssh executed failed", e); - throw new SshException("Ssh executed failed " + e.getMessage()); + s_logger.debug("SSH execution failed", e); + throw new SshException("SSH execution failed " + e.getMessage()); } finally { if (sshSession != null) sshSession.close(); From 40d9258986ac0a49d00528e06f710be77b0fffc8 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 14 Jan 2022 15:28:26 +0530 Subject: [PATCH 02/22] support for private registry - containerd --- engine/schema/pom.xml | 12 +- .../cluster/KubernetesClusterManagerImpl.java | 123 ++++++++---------- ...esClusterResourceModifierActionWorker.java | 106 +++++++-------- .../KubernetesClusterStartWorker.java | 37 +++--- .../cluster/CreateKubernetesClusterCmd.java | 22 +--- .../resources/conf/k8s-control-node-add.yml | 18 +++ .../main/resources/conf/k8s-control-node.yml | 18 +++ .../src/main/resources/conf/k8s-node.yml | 18 +++ .../views/compute/CreateKubernetesCluster.vue | 8 -- 9 files changed, 188 insertions(+), 174 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 588ed92a0d87..70556b61fabc 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,7 +122,8 @@ wget - https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -181,7 +182,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -193,7 +195,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -205,7 +208,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index aec96fe67785..550ac74df4b3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,57 +16,6 @@ // under the License. package com.cloud.kubernetes.cluster; -import java.math.BigInteger; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.annotation.AnnotationService; -import org.apache.cloudstack.annotation.dao.AnnotationDao; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; -import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; -import org.apache.cloudstack.api.response.KubernetesClusterResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; @@ -165,6 +114,53 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -428,21 +424,18 @@ private boolean validateServiceOffering(final ServiceOffering serviceOffering, f private void validateDockerRegistryParams(final String dockerRegistryUserName, final String dockerRegistryPassword, - final String dockerRegistryUrl, - final String dockerRegistryEmail) { + final String dockerRegistryUrl) { // if no params related to docker registry specified then nothing to validate so return true if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { + (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty())) { return; } // all params related to docker registry must be specified or nothing if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { + (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()))) { throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); } @@ -451,12 +444,6 @@ private void validateDockerRegistryParams(final String dockerRegistryUserName, } catch (MalformedURLException e) { throw new InvalidParameterValueException("Invalid docker registry url specified"); } - - Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); - Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); - if (!matcher.find()) { - throw new InvalidParameterValueException("Invalid docker registry email specified"); - } } private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { @@ -619,7 +606,6 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); @@ -706,7 +692,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl); Network network = null; if (networkId != null) { @@ -729,7 +715,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu } if (!KubernetesClusterExperimentalFeaturesEnabled.value() && (!Strings.isNullOrEmpty(dockerRegistryUrl) || - !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryEmail) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { + !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { throw new CloudRuntimeException(String.format("Private registry for the Kubernetes cluster is an experimental feature. Use %s configuration for enabling experimental features", KubernetesClusterExperimentalFeaturesEnabled.key())); } } @@ -779,7 +765,6 @@ private void addKubernetesClusterDetails(final KubernetesCluster kubernetesClust final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final boolean networkCleanup = cmd.getNetworkId() == null; Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -797,9 +782,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); } - if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); - } + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); SecureRandom random = new SecureRandom(); String randomPassword = new BigInteger(130, random).toString(32); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 595aab939994..698ee3176e97 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,26 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; @@ -74,9 +54,9 @@ import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.LaunchPermissionDao; -import com.cloud.storage.VolumeApiService; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.SSHKeyPairVO; @@ -97,6 +77,24 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -164,56 +162,48 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to + + k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + return k8sNodeConfig; + } + + protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { + /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to * use docker private registry */ - String dockerUserName = null; - String dockerPassword = null; - String dockerRegistryUrl = null; - String dockerRegistryEmail = null; + String registryUsername = null; + String registryPassword = null; + String registryUrl = null; + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); for (KubernetesClusterDetailsVO detail : details) { if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - dockerUserName = detail.getValue(); + registryUsername = detail.getValue(); } if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - dockerPassword = detail.getValue(); + registryPassword = detail.getValue(); } if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - dockerRegistryUrl = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { - dockerRegistryEmail = detail.getValue(); + registryUrl = detail.getValue(); } } - if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { - // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section - // optional or conditionally applied - String dockerConfigString = "write_files:\n" + - " - path: /.docker/config.json\n" + - " owner: core:core\n" + - " permissions: '0644'\n" + - " content: |\n" + - " {\n" + - " \"auths\": {\n" + - " {{docker.url}}: {\n" + - " \"auth\": {{docker.secret}},\n" + - " \"email\": {{docker.email}}\n" + - " }\n" + - " }\n" + - " }"; - k8sNodeConfig = k8sNodeConfig.replace("write_files:", dockerConfigString); - final String dockerUrlKey = "{{docker.url}}"; - final String dockerAuthKey = "{{docker.secret}}"; - final String dockerEmailKey = "{{docker.email}}"; - final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; + if (!Strings.isNullOrEmpty(registryUsername) && !Strings.isNullOrEmpty(registryPassword) && !Strings.isNullOrEmpty(registryUrl)) { + // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details + String runCmd = "- bash -x /opt/bin/setup-containerd"; + + String registryEp = registryUrl.split("://")[1]; + k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + + final String usernamePasswordKey = registryUsername + ":" + registryPassword; String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); - k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); } - return k8sNodeConfig; + return k8sConfig; } - protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index b6da75dba81b..c3d59680a9cf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -17,24 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.IOException; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.framework.ca.Certificate; -import org.apache.cloudstack.utils.security.CertUtils; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.dc.DataCenter; import com.cloud.dc.Vlan; import com.cloud.dc.VlanVO; @@ -75,6 +57,23 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; import com.google.common.base.Strings; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -175,6 +174,7 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; } @@ -237,6 +237,7 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 376c148abb7a..d767c2b52bf0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -16,10 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; -import java.security.InvalidParameterException; - -import javax.inject.Inject; - +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -41,10 +41,8 @@ import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; -import com.cloud.kubernetes.cluster.KubernetesClusterService; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.security.InvalidParameterException; @APICommand(name = CreateKubernetesClusterCmd.APINAME, description = "Creates a Kubernetes cluster", @@ -140,10 +138,6 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "URL for the docker image private registry") private String dockerRegistryUrl; - @Parameter(name = ApiConstants.DOCKER_REGISTRY_EMAIL, type = CommandType.STRING, - description = "email of the docker image private registry user") - private String dockerRegistryEmail; - @Parameter(name = ApiConstants.NODE_ROOT_DISK_SIZE, type = CommandType.LONG, description = "root disk size in GB for each node") private Long nodeRootDiskSize; @@ -226,10 +220,6 @@ public String getDockerRegistryUrl() { return dockerRegistryUrl; } - public String getDockerRegistryEmail() { - return dockerRegistryEmail; - } - public Long getNodeRootDiskSize() { if (nodeRootDiskSize != null) { if (nodeRootDiskSize < DEFAULT_NODE_ROOT_DISK_SIZE) { diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index db295188eba5..5310e52caf61 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -213,6 +213,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -242,6 +259,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 730477bd4eef..5a4f8b3825d7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -270,6 +270,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -299,6 +316,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index eea369921ac8..e6ffed888f6b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -213,6 +213,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -242,5 +259,6 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 6b6b81f67918..52aa831d75c5 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -226,14 +226,6 @@ }]" :placeholder="apiParams.dockerregistryurl.description"/> - - - -
From 36ebfc256665469fff5d576f52e4dd4952b5d226 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 17 Jan 2022 13:45:20 +0530 Subject: [PATCH 03/22] Enable updating template type (only) for system owned templates via UI --- ui/src/config/section/image.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 0a4667efd608..e98318aca103 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -100,12 +100,14 @@ export default { return (['Admin'].includes(store.userInfo.roletype) || // If admin or owner or belongs to current project (record.domainid === store.userInfo.domainid && record.account === store.userInfo.account) || (record.domainid === store.userInfo.domainid && record.projectid && store.project && store.project.id && record.projectid === store.project.id)) && - record.templatetype !== 'SYSTEM' && record.isready }, args: (record, store) => { var fields = ['name', 'displaytext', 'passwordenabled', 'ostypeid', 'isdynamicallyscalable'] if (['Admin'].includes(store.userInfo.roletype)) { + if (record.templatetype === 'SYSTEM') { + fields = [] + } fields.push('templatetype') } return fields From a23be850460397bf5564ae5a59fae33a8c3da540 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 09:58:46 +0530 Subject: [PATCH 04/22] edit indents --- .../upgrade/SystemVmTemplateRegistration.java | 4 +- ...esClusterResourceModifierActionWorker.java | 1 + .../KubernetesClusterStartWorker.java | 2 + .../resources/conf/k8s-control-node-add.yml | 30 +++++++------- .../main/resources/conf/k8s-control-node.yml | 29 +++++++------- .../src/main/resources/conf/k8s-node.yml | 30 +++++++------- .../util/create-kubernetes-binaries-iso.sh | 2 +- .../debian/opt/cloud/bin/setup/cksnode.sh | 11 ++++-- .../scripts/configure_systemvm_services.sh | 2 +- .../main/java/com/cloud/utils/DigestUtil.java | 39 +++++++++++++++++++ .../java/com/cloud/utils/EncryptionUtil.java | 16 -------- 11 files changed, 98 insertions(+), 68 deletions(-) create mode 100644 utils/src/main/java/com/cloud/utils/DigestUtil.java diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 4119b11ae508..6a927d4d2fec 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,7 +36,7 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; -import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.DigestUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -726,7 +726,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = EncryptionUtil.calculateChecksum(tempFile); + String templateChecksum = DigestUtil.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 698ee3176e97..b624c0770a57 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -164,6 +164,7 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + return k8sNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index c3d59680a9cf..f0aa2a410ec4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -175,6 +175,7 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } @@ -238,6 +239,7 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 5310e52caf61..877eaf72cb25 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -214,21 +214,21 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 5a4f8b3825d7..02e64093b08a 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -271,21 +271,22 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - echo "Restarting containerd service" - systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index e6ffed888f6b..9ec455e35eb7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -214,21 +214,21 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index aec95241eb96..ce7626c98c22 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -136,7 +136,7 @@ while read -r line; do sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" - sudo ctr image rm "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index 240e15f95ff2..169f30ce06a8 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -30,10 +30,15 @@ setup_k8s_node() { sed -i 's/3922/22/g' /etc/ssh/sshd_config # Prevent root login - > /root/.ssh/authorized_keys - passwd -l root +# > /root/.ssh/authorized_keys +# passwd -l root #sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd + # Update containerd configuration + mkdir -p /etc/containerd + containerd config default>/etc/containerd/config.toml + systemctl restart containerd + swapoff -a sudo sed -i '/ swap / s/^/#/' /etc/fstab log_it "Swap disabled" @@ -61,8 +66,6 @@ setup_k8s_node() { log_it "Starting cloud-init services" systemctl enable --now --no-block containerd - systemctl enable --now --no-block docker.socket - systemctl enable --now --no-block docker.service if [ -f /home/cloud/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 8c844b893b53..67bb03491e49 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -19,7 +19,7 @@ set -e set -x -CLOUDSTACK_RELEASE=4.16.0 +CLOUDSTACK_RELEASE=4.16.1 function configure_apache2() { # Enable ssl, rewrite and auth diff --git a/utils/src/main/java/com/cloud/utils/DigestUtil.java b/utils/src/main/java/com/cloud/utils/DigestUtil.java new file mode 100644 index 000000000000..828af672379a --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/DigestUtil.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.utils; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; + +public class DigestUtil { + public static final Logger s_logger = Logger.getLogger(DigestUtil.class.getName()); + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } +} diff --git a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java index 7dbf13b54447..37cea3499d99 100644 --- a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java +++ b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java @@ -20,19 +20,13 @@ import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.jasypt.encryption.pbe.PBEStringEncryptor; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -74,14 +68,4 @@ public static String generateSignature(String data, String key) { throw new CloudRuntimeException("unable to generate signature", e); } } - - public static String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - s_logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } } From 10239449b185a6e20ecadee184284f77c42fde0b Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 15:37:46 +0530 Subject: [PATCH 05/22] Address comments and move cmd from patch file to cloud-init runcmd --- engine/schema/pom.xml | 12 ++---- .../upgrade/SystemVmTemplateRegistration.java | 10 ++++- .../resources/conf/k8s-control-node-add.yml | 1 + .../main/resources/conf/k8s-control-node.yml | 1 + .../src/main/resources/conf/k8s-node.yml | 1 + .../debian/opt/cloud/bin/setup/cksnode.sh | 9 +---- .../main/java/com/cloud/utils/DigestUtil.java | 39 ------------------- .../utils/security/DigestHelper.java | 17 +++++++- 8 files changed, 33 insertions(+), 57 deletions(-) delete mode 100644 utils/src/main/java/com/cloud/utils/DigestUtil.java diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 70556b61fabc..588ed92a0d87 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,8 +122,7 @@ wget - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt + https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -182,8 +181,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -195,8 +193,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -208,8 +205,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 6a927d4d2fec..214243886997 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,7 +36,6 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; -import com.cloud.utils.DigestUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -55,6 +54,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -679,6 +679,12 @@ public void registerTemplate(Pair hypervisorA } } + /** + * This method parses the metadata file consisting of the systemVM templates information + * @return the version of the systemvm template that is to be used. This is done to in order + * to fallback on the latest available version of the systemVM template when there does not + * exist a template corresponding to the current code version. + */ public static String parseMetadataFile() { try { Ini ini = new Ini(); @@ -726,7 +732,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = DigestUtil.calculateChecksum(tempFile); + String templateChecksum = DigestHelper.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 877eaf72cb25..0ca165fa6bee 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -260,6 +260,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 02e64093b08a..5087ec262dde 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -318,6 +318,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 9ec455e35eb7..c2cc52133bfc 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -260,5 +260,6 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index 169f30ce06a8..ac672a265399 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -30,15 +30,10 @@ setup_k8s_node() { sed -i 's/3922/22/g' /etc/ssh/sshd_config # Prevent root login -# > /root/.ssh/authorized_keys -# passwd -l root + > /root/.ssh/authorized_keys + passwd -l root #sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd - # Update containerd configuration - mkdir -p /etc/containerd - containerd config default>/etc/containerd/config.toml - systemctl restart containerd - swapoff -a sudo sed -i '/ swap / s/^/#/' /etc/fstab log_it "Swap disabled" diff --git a/utils/src/main/java/com/cloud/utils/DigestUtil.java b/utils/src/main/java/com/cloud/utils/DigestUtil.java deleted file mode 100644 index 828af672379a..000000000000 --- a/utils/src/main/java/com/cloud/utils/DigestUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.utils; - -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; - -public class DigestUtil { - public static final Logger s_logger = Logger.getLogger(DigestUtil.class.getName()); - public static String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - s_logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } -} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java index f856b1f2892d..a8307cc751fe 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java @@ -16,18 +16,23 @@ // under the License. package org.apache.cloudstack.utils.security; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; public class DigestHelper { - + public static final Logger s_logger = Logger.getLogger(DigestHelper.class.getName()); public static ChecksumValue digest(String algorithm, InputStream is) throws NoSuchAlgorithmException, IOException { MessageDigest digest = MessageDigest.getInstance(algorithm); ChecksumValue checksum = null; @@ -131,4 +136,14 @@ protected static boolean isAlgorithmPresent(String checksum) { public static String getHashValueFromChecksumValue(String checksum) { return isAlgorithmPresent(checksum) ? new ChecksumValue(checksum).getChecksum() : checksum; } + + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } } From 69b4e925451db4539d97300d74efcd1be9dde823 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 17:00:19 +0530 Subject: [PATCH 06/22] temporary change --- engine/schema/pom.xml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 588ed92a0d87..70556b61fabc 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,7 +122,8 @@ wget - https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -181,7 +182,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -193,7 +195,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -205,7 +208,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} From fa634ab91f61a9843af0188677fef1483e3148e2 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 21 Jan 2022 12:08:11 +0530 Subject: [PATCH 07/22] update k8s test to use k8s version 1.21.5 (instead of 1.21.3 - due to https://github.com/kubernetes/kubernetes/pull/104530) --- .../upgrade/SystemVmTemplateRegistration.java | 53 +++++++++---------- .../smoke/test_kubernetes_clusters.py | 20 +++---- 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 214243886997..1eb3fdd20bbc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -360,24 +360,24 @@ public Long getRegisteredTemplateId(Pair hype } private static String fetchTemplatesPath() { - String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; + String filePath = RELATIVE_TEMPLATE_PATH + METADATA_FILE_NAME; + LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); + File metaFile = new File(filePath); + String templatePath = null; + if (metaFile.exists()) { + templatePath = RELATIVE_TEMPLATE_PATH; + } + if (templatePath == null) { + filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME; + metaFile = new File(filePath); + templatePath = ABSOLUTE_TEMPLATE_PATH; LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); - File metaFile = new File(filePath); - String templatePath = null; - if (metaFile.exists()) { - templatePath = RELATIVE_TEMPLATE_PATH; - } - if (templatePath == null) { - filePath = ABSOLUTE_TEMPLATE_PATH + METADATA_FILE_NAME; - metaFile = new File(filePath); - templatePath = ABSOLUTE_TEMPLATE_PATH; - LOGGER.debug(String.format("Looking for file [ %s ] in the classpath.", filePath)); - if (!metaFile.exists()) { - String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); - LOGGER.error(errMsg); - throw new CloudRuntimeException(errMsg); - } + if (!metaFile.exists()) { + String errMsg = String.format("Unable to locate metadata file in your setup at %s", filePath.toString()); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); } + } return templatePath; } @@ -470,7 +470,7 @@ private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { template.setCrossZones(true); template.setHypervisorType(details.getHypervisorType()); template.setState(VirtualMachineTemplate.State.Inactive); - template.setDeployAsIs(Hypervisor.HypervisorType.VMware.equals(details.getHypervisorType())); + template.setDeployAsIs(false); template = vmTemplateDao.persist(template); return template; } @@ -566,12 +566,12 @@ private static void readTemplateProperties(String path, SystemVMTemplateDetails } private void updateTemplateTablesOnFailure(long templateId) { - VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); - template.setState(VirtualMachineTemplate.State.Inactive); - vmTemplateDao.update(template.getId(), template); - vmTemplateDao.remove(templateId); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); - templateDataStoreDao.remove(templateDataStoreVO.getId()); + VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); + template.setState(VirtualMachineTemplate.State.Inactive); + vmTemplateDao.update(template.getId(), template); + vmTemplateDao.remove(templateId); + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); + templateDataStoreDao.remove(templateDataStoreVO.getId()); } public static void unmountStore(String filePath) { @@ -592,7 +592,7 @@ public static void unmountStore(String filePath) { } private void setupTemplate(String templateName, Pair hypervisorAndTemplateName, - String destTempFolder) throws CloudRuntimeException { + String destTempFolder) throws CloudRuntimeException { String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt"); if (setupTmpltScript == null) { throw new CloudRuntimeException("Unable to find the createtmplt.sh"); @@ -640,7 +640,7 @@ private Long performTemplateRegistrationOperations(Pair hypervisorAndTemplateName, - Pair storeUrlAndId, VMTemplateVO templateVO, String filePath) { + Pair storeUrlAndId, VMTemplateVO templateVO, String filePath) { Long templateId = null; try { templateId = templateVO.getId(); @@ -828,9 +828,6 @@ private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { templateVO.setUrl(NewTemplateUrl.get(hypervisorAndTemplateName.getKey())); templateVO.setChecksum(NewTemplateChecksum.get(hypervisorAndTemplateName.getKey())); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", hypervisorAndTemplateName.getKey().name()); diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 0387ddf9cc1b..46f1f5d02bdc 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -101,12 +101,12 @@ def setUpClass(cls): (cls.services["cks_kubernetes_versions"]["1.20.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.20.9"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_1_21_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.3"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_3.id) + cls.kubernetes_version_1_21_5 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.5"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_5.id) except Exception as e: cls.setup_failed = True cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.21.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.3"]["url"], e)) + (cls.services["cks_kubernetes_versions"]["1.21.5"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.5"]["url"], e)) if cls.setup_failed == False: cks_offering_data = cls.services["cks_service_offering"] @@ -349,7 +349,7 @@ def test_01_invalid_upgrade_kubernetes_cluster(self): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Downgrading Kubernetes cluster with ID: %s to a lower version. This should fail!" % k8s_cluster.id) @@ -362,7 +362,7 @@ def test_01_invalid_upgrade_kubernetes_cluster(self): self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -381,12 +381,12 @@ def test_02_upgrade_kubernetes_cluster(self): time.sleep(self.services["sleep"]) self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -434,7 +434,7 @@ def test_04_autoscale_kubernetes_cluster(self): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Autoscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) try: @@ -535,12 +535,12 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return From 1053b5081c0711ad24de1e3dfb54ac89abf3bf81 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 14 Jan 2022 11:41:54 +0530 Subject: [PATCH 08/22] This PR/commit comprises of the following: - Support to fallback on the older systemVM template in case of no change in template across ACS versions - Update core user to cloud in CKS - Display details of accessing CKS nodes in the UI - K8s Access tab - Update systemvm template from debian 11 to debian 11.2 - Update letsencrypt cert - Remove docker dependency as from ACS 4.16 onward k8s has deprecated support for docker - use containerd as container runtime --- .gitignore | 1 + .../main/java/com/cloud/agent/AgentShell.java | 50 +++++++++---------- engine/schema/pom.xml | 8 +-- .../cloud/upgrade/DatabaseUpgradeChecker.java | 9 ++-- .../upgrade/SystemVmTemplateRegistration.java | 22 ++------ .../META-INF/db/schema-41600to41610.sql | 3 ++ engine/schema/templateConfig.sh | 20 +++++--- .../KubernetesClusterActionWorker.java | 42 ++++++++-------- .../resources/conf/k8s-control-node-add.yml | 21 ++++---- .../main/resources/conf/k8s-control-node.yml | 23 +++++---- .../src/main/resources/conf/k8s-node.yml | 21 ++++---- .../resources/script/upgrade-kubernetes.sh | 2 +- pom.xml | 1 + .../util/create-kubernetes-binaries-iso.sh | 22 ++++---- .../debian/opt/cloud/bin/setup/cksnode.sh | 2 +- .../scripts/configure_systemvm_services.sh | 13 +++-- .../scripts/install_systemvm_packages.sh | 2 +- .../appliance/systemvmtemplate/template.json | 4 +- ui/public/locales/en.json | 2 + ui/src/views/compute/KubernetesServiceTab.vue | 3 ++ .../java/com/cloud/utils/EncryptionUtil.java | 30 ++++++++--- .../java/com/cloud/utils/script/Script.java | 19 ++++--- .../com/cloud/utils/ssh/SSHCmdHelper.java | 15 +++--- 23 files changed, 175 insertions(+), 160 deletions(-) diff --git a/.gitignore b/.gitignore index b67dc8e6499f..011a4b005dce 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ tools/cli/cloudmonkey/precache.py tools/marvin/marvin/cloudstackAPI/ tools/marvin/build/ tools/cli/build/ +tools/appliance/systemvmtemplate/packer_cache/ *.jar *.war *.mar diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index 01654ac9caa9..0ab5b7863a85 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -16,6 +16,26 @@ // under the License. package com.cloud.agent; +import com.cloud.agent.Agent.ExitStatus; +import com.cloud.agent.dao.StorageComponent; +import com.cloud.agent.dao.impl.PropertiesStorage; +import com.cloud.resource.ServerResource; +import com.cloud.utils.LogUtils; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.ProcessUtil; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.backoff.BackoffAlgorithm; +import com.cloud.utils.backoff.impl.ConstantTimeBackoff; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.common.base.Strings; +import org.apache.commons.daemon.Daemon; +import org.apache.commons.daemon.DaemonContext; +import org.apache.commons.daemon.DaemonInitException; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.DOMConfigurator; + +import javax.naming.ConfigurationException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -30,28 +50,6 @@ import java.util.Properties; import java.util.UUID; -import javax.naming.ConfigurationException; - -import org.apache.commons.daemon.Daemon; -import org.apache.commons.daemon.DaemonContext; -import org.apache.commons.daemon.DaemonInitException; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.xml.DOMConfigurator; - -import com.cloud.agent.Agent.ExitStatus; -import com.cloud.agent.dao.StorageComponent; -import com.cloud.agent.dao.impl.PropertiesStorage; -import com.cloud.resource.ServerResource; -import com.cloud.utils.LogUtils; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.ProcessUtil; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.backoff.BackoffAlgorithm; -import com.cloud.utils.backoff.impl.ConstantTimeBackoff; -import com.cloud.utils.exception.CloudRuntimeException; -import com.google.common.base.Strings; - public class AgentShell implements IAgentShell, Daemon { private static final Logger s_logger = Logger.getLogger(AgentShell.class.getName()); @@ -423,13 +421,13 @@ private void launchAgentFromClassInfo(String resourceClassNames) throws Configur } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { - throw new ConfigurationException("Security excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Security exception when loading resource: " + name + " due to: " + e.toString()); } catch (final NoSuchMethodException e) { - throw new ConfigurationException("Method not found excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Method not found exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalArgumentException e) { - throw new ConfigurationException("Illegal argument excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Illegal argument exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InstantiationException e) { - throw new ConfigurationException("Instantiation excetion when loading resource: " + name + " due to: " + e.toString()); + throw new ConfigurationException("Instantiation exception when loading resource: " + name + " due to: " + e.toString()); } catch (final IllegalAccessException e) { throw new ConfigurationException("Illegal access exception when loading resource: " + name + " due to: " + e.toString()); } catch (final InvocationTargetException e) { diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 867535251547..588ed92a0d87 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -73,10 +73,10 @@ - def projectVersion = project.version + def projectVersion = project.properties['project.systemvm.template.version'] String[] versionParts = projectVersion.tokenize('.') - pom.properties['cs.version'] = "4.16" - pom.properties['patch.version'] = "0" + pom.properties['cs.version'] = versionParts[0] + "." + versionParts[1] + pom.properties['patch.version'] = versionParts[2] @@ -146,7 +146,7 @@ bash templateConfig.sh - ${project.version} + ${project.systemvm.template.version} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index cf3f7282f9f6..3c6abbc6e86d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -367,10 +367,11 @@ public void check() { return; } - SystemVmTemplateRegistration.parseMetadataFile(); - final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); - SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(currentVersion.getMajorRelease()) + "." + String.valueOf(currentVersion.getMinorRelease()); - SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(currentVersion.getPatchRelease()); + String csVersion = SystemVmTemplateRegistration.parseMetadataFile(); + final CloudStackVersion sysVmVersion = CloudStackVersion.parse(csVersion); + final CloudStackVersion currentVersion = CloudStackVersion.parse(currentVersionValue); + SystemVmTemplateRegistration.CS_MAJOR_VERSION = String.valueOf(sysVmVersion.getMajorRelease()) + "." + String.valueOf(sysVmVersion.getMinorRelease()); + SystemVmTemplateRegistration.CS_TINY_VERSION = String.valueOf(sysVmVersion.getPatchRelease()); s_logger.info("DB version = " + dbVersion + " Code Version = " + currentVersion); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index d788adac8244..4119b11ae508 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,6 +36,7 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; +import com.cloud.utils.EncryptionUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -54,7 +55,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -64,7 +64,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; @@ -351,16 +350,6 @@ public static boolean validateIfSeeded(String url, String path) { } } - private String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - LOGGER.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } - public Long getRegisteredTemplateId(Pair hypervisorAndTemplateName) { VMTemplateVO vmTemplate = vmTemplateDao.findLatestTemplateByName(hypervisorAndTemplateName.second()); Long templateId = null; @@ -690,7 +679,7 @@ public void registerTemplate(Pair hypervisorA } } - public static void parseMetadataFile() { + public static String parseMetadataFile() { try { Ini ini = new Ini(); ini.load(new FileReader(METADATA_FILE)); @@ -702,6 +691,8 @@ public static void parseMetadataFile() { NewTemplateChecksum.put(hypervisorType, section.get("checksum")); NewTemplateUrl.put(hypervisorType, section.get("downloadurl")); } + Ini.Section section = ini.get("default"); + return section.get("version"); } catch (Exception e) { String errMsg = String.format("Failed to parse systemVM template metadata file: %s", METADATA_FILE); LOGGER.error(errMsg, e); @@ -735,7 +726,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = calculateChecksum(tempFile); + String templateChecksum = EncryptionUtil.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; @@ -812,9 +803,6 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index bf9cbf861d73..979d87d461e7 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -127,3 +127,6 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id -- Add PK to cloud_usage.usage_vpn_user CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_vpn_user', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)'); + +UPDATE `cloud`.`vm_template` set deploy_as_is = 0 where id = 8; + diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index c3093530e7b2..891c73db1bbf 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -23,8 +23,10 @@ function getTemplateVersion() { subversion1="$(cut -d'.' -f1 <<<"$version")" subversion2="$(cut -d'.' -f2 <<<"$version")" minorversion="$(cut -d'.' -f3 <<<"$version")" + securityversion="$(cut -d'.' -f4 <<<"$version")" export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" + export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" } function getGenericName() { @@ -52,12 +54,14 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) + echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE for i in "${!templates[@]}" do section="$i" hvName=$(getGenericName $i) - templatename="systemvm-${i}-${CS_VERSION}.${CS_MINOR_VERSION}" - checksum=$(getChecksum "$fileData" $hvName) + + templatename="systemvm-${i}-${VERSION}" + checksum=$(getChecksum "$fileData" "$VERSION-$hvName") downloadurl="${templates[$i]}" filename=$(echo ${downloadurl##*'/'}) echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\n" >> $METADATAFILE @@ -66,12 +70,12 @@ function createMetadataFile() { declare -A templates getTemplateVersion $1 -templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-kvm.qcow2.bz2" - ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-${CS_VERSION}.${CS_MINOR_VERSION}-vmware.ova" - ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-xen.vhd.bz2" - ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-hyperv.vhd.zip" - ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-kvm.qcow2.bz2" - ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$CS_VERSION.$CS_MINOR_VERSION-ovm.raw.bz2" ) +templates=( ["kvm"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["vmware"]="https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-vmware.ova" + ["xenserver"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-xen.vhd.bz2" + ["hyperv"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-hyperv.vhd.zip" + ["lxc"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-kvm.qcow2.bz2" + ["ovm3"]="https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-ovm.raw.bz2" ) PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 62d45a3e028d..cf0403b4000a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -17,27 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Objects; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.ca.CAManager; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.VlanDao; @@ -82,10 +61,29 @@ import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; public class KubernetesClusterActionWorker { - public static final String CLUSTER_NODE_VM_USER = "core"; + public static final String CLUSTER_NODE_VM_USER = "cloud"; public static final int CLUSTER_API_PORT = 6443; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 0891d2015b30..db295188eba5 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +192,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +210,16 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,6 +241,7 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 322f08099cf7..730477bd4eef 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -51,7 +51,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -138,7 +138,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -187,7 +187,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -198,7 +198,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -216,7 +216,7 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -231,7 +231,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -267,16 +267,16 @@ write_files: /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -298,6 +298,7 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 28ba43e75845..eea369921ac8 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -18,7 +18,7 @@ --- users: - - name: core + - name: cloud sudo: ALL=(ALL) NOPASSWD:ALL shell: /bin/bash ssh_authorized_keys: @@ -31,7 +31,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -118,7 +118,7 @@ write_files: fi retval=0 set +e - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -165,7 +165,7 @@ write_files: fi systemctl enable kubelet && systemctl start kubelet - modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then crucial_cmd_attempts=1 @@ -176,7 +176,7 @@ write_files: fi retval=0 set +e - kubeadm config images pull + kubeadm config images pull --cri-socket /run/containerd/containerd.sock retval=$? set -e if [ $retval -eq 0 ]; then @@ -192,7 +192,7 @@ write_files: content: | #!/bin/bash -e - if [[ -f "/home/core/success" ]]; then + if [[ -f "/home/cloud/success" ]]; then echo "Already provisioned!" exit 0 fi @@ -210,16 +210,16 @@ write_files: fi kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - sudo touch /home/core/success - echo "true" > /home/core/success + sudo touch /home/cloud/success + echo "true" > /home/cloud/success - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] Type=simple @@ -241,5 +241,6 @@ write_files: ExecStart=/opt/bin/deploy-kube-system runcmd: + - chown -R cloud:cloud /home/cloud/.ssh - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 7e0c3c00a8a5..b85ea00e9ed6 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do - docker load < "${BINARIES_DIR}/docker/$line" + ctr image import "${BINARIES_DIR}/docker/$line" done <<< "$output" fi if [ -e "${BINARIES_DIR}/provider.yaml" ]; then diff --git a/pom.xml b/pom.xml index 67cbc6e2e5bd..6813bb867c3e 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,7 @@ UTF-8 UTF-8 + 4.16.1.0 11 diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index ba3dca77a8d2..aec95241eb96 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -98,19 +98,18 @@ provider_conf_file="${working_dir}/provider.yaml" curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} echo "Fetching k8s docker images..." -docker -v +ctr -v if [ $? -ne 0 ]; then - echo "Installing docker..." + echo "Installing containerd..." if [ -f /etc/redhat-release ]; then sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm - sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce - sudo systemctl enable docker && sudo systemctl start docker + sudo yum install -y containerd.io elif [ -f /etc/lsb-release ]; then - sudo apt update && sudo apt install docker.io -y - sudo systemctl enable docker && sudo systemctl start docker + sudo apt update && sudo apt install containerd.io -y fi + sudo systemctl enable containerd && sudo systemctl start containerd fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list --kubernetes-version=${RELEASE}` @@ -130,11 +129,14 @@ provider_image=`grep "image:" ${provider_conf_file} | cut -d ':' -f2- | tr -d ' output=`printf "%s\n" ${output} ${provider_image}` while read -r line; do - echo "Downloading docker image $line ---" - sudo docker pull "$line" + echo "Downloading image $line ---" + if [[ $line == kubernetesui* ]] || [[ $line == apache* ]]; then + line="docker.io/${line}" + fi + sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` - sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" - sudo docker image rm "$line" + sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index a864d188d009..240e15f95ff2 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -63,7 +63,7 @@ setup_k8s_node() { systemctl enable --now --no-block containerd systemctl enable --now --no-block docker.socket systemctl enable --now --no-block docker.service - if [ -f /home/core/success ]; then + if [ -f /home/cloud/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final else diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index db3eec5cb40c..8c844b893b53 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -50,9 +50,12 @@ function configure_cacerts() { CDIR=$(pwd) cd /tmp # Add LetsEncrypt ca-cert - wget https://letsencrypt.org/certs/lets-encrypt-x3-cross-signed.der - keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx3cross -file lets-encrypt-x3-cross-signed.der - rm -f lets-encrypt-x3-cross-signed.der + wget https://letsencrypt.org/certs/lets-encrypt-r3.der + wget https://letsencrypt.org/certs/isrgrootx1.der + + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityr3 -file lets-encrypt-r3.der + keytool -trustcacerts -keystore /etc/ssl/certs/java/cacerts -storepass changeit -noprompt -importcert -alias letsencryptauthorityx1 -file isrgrootx1.der + rm -f lets-encrypt-r3.der isrgrootx1.der cd $CDIR } @@ -126,10 +129,6 @@ function configure_services() { # Disable container services systemctl disable containerd - systemctl disable docker.service - systemctl stop docker.service - systemctl disable docker.socket - systemctl stop docker.socket # Disable cloud init by default cat < /etc/cloud/cloud.cfg.d/cloudstack.cfg diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index 555a00c28edb..3b4ef6537286 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -98,7 +98,7 @@ function install_packages() { apt-key fingerprint 0EBFCD88 add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" apt-get update - ${apt_get} install docker-ce docker-ce-cli containerd.io + ${apt_get} install containerd.io apt_clean diff --git a/tools/appliance/systemvmtemplate/template.json b/tools/appliance/systemvmtemplate/template.json index bd932bf6cdef..31eb2ea5abf1 100644 --- a/tools/appliance/systemvmtemplate/template.json +++ b/tools/appliance/systemvmtemplate/template.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:5f6aed67b159d7ccc1a90df33cc8a314aa278728a6f50707ebf10c02e46664e383ca5fa19163b0a1c6a4cb77a39587881584b00b45f512b4a470f1138eaa1801", - "iso_url": "https://cdimage.debian.org/debian-cd/11.0.0/amd64/iso-cd/debian-11.0.0-amd64-netinst.iso", + "iso_checksum": "sha512:c685b85cf9f248633ba3cd2b9f9e781fa03225587e0c332aef2063f6877a1f0622f56d44cf0690087b0ca36883147ecb5593e3da6f965968402cdbdf12f6dd74", + "iso_url": "https://cdimage.debian.org/debian-cd/11.2.0/amd64/iso-cd/debian-11.2.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 9c44d838495c..8ffc9d7892e3 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,7 @@ "label.accept.project.invitation": "Accept project invitation", "label.access": "Access", "label.accesskey": "Access Key", +"label.acess.kubernetes.nodes": "Access Kubernetes Nodes", "label.account": "Account", "label.account.and.security.group": "Account - Security group", "label.account.details": "Account details", @@ -1232,6 +1233,7 @@ "label.keypair": "SSH Key Pair", "label.kubeconfig.cluster": "Kubernetes Cluster Config", "label.kubernetes": "Kubernetes", +"label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using:
ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address]

where,
ssh_key: points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server.
port_number: can be obtained from the Port Forwarding Tab (Public Port column)", "label.kubernetes.cluster": "Kubernetes cluster", "label.kubernetes.cluster.create": "Create Kubernetes Cluster", "label.kubernetes.cluster.delete": "Delete Kubernetes Cluster", diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 97f1c5456ece..039d09eab87e 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -88,6 +88,9 @@

{{ $t('label.more.access.dashboard.ui') }}, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui

+ +

+ { private static final Logger s_logger = Logger.getLogger(Script.class); @@ -320,7 +319,7 @@ public String call() { try { _logger.trace("Checking exit value of process"); _process.exitValue(); - _logger.trace("Script ran within the alloted time"); + _logger.trace("Script ran within the allotted time"); } catch (IllegalThreadStateException e) { _logger.warn("Interrupting script."); _isTimeOut = true; diff --git a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java index 291f8c1b43fd..dc4333be6a1a 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SSHCmdHelper.java @@ -19,16 +19,15 @@ package com.cloud.utils.ssh; -import java.io.IOException; -import java.io.InputStream; - +import com.google.common.base.Strings; +import com.trilead.ssh2.ChannelCondition; +import com.trilead.ssh2.Session; import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; -import com.google.common.base.Strings; -import com.trilead.ssh2.ChannelCondition; -import com.trilead.ssh2.Session; +import java.io.IOException; +import java.io.InputStream; public class SSHCmdHelper { private static final Logger s_logger = Logger.getLogger(SSHCmdHelper.class); @@ -242,8 +241,8 @@ public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshC } return result; } catch (Exception e) { - s_logger.debug("Ssh executed failed", e); - throw new SshException("Ssh executed failed " + e.getMessage()); + s_logger.debug("SSH execution failed", e); + throw new SshException("SSH execution failed " + e.getMessage()); } finally { if (sshSession != null) sshSession.close(); From b102259f157bdd6cd5e8426df8dc91fbd1a276db Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 14 Jan 2022 15:28:26 +0530 Subject: [PATCH 09/22] support for private registry - containerd --- engine/schema/pom.xml | 12 +- .../cluster/KubernetesClusterManagerImpl.java | 123 ++++++++---------- ...esClusterResourceModifierActionWorker.java | 106 +++++++-------- .../KubernetesClusterStartWorker.java | 37 +++--- .../cluster/CreateKubernetesClusterCmd.java | 22 +--- .../resources/conf/k8s-control-node-add.yml | 18 +++ .../main/resources/conf/k8s-control-node.yml | 18 +++ .../src/main/resources/conf/k8s-node.yml | 18 +++ .../views/compute/CreateKubernetesCluster.vue | 8 -- 9 files changed, 188 insertions(+), 174 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 588ed92a0d87..70556b61fabc 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,7 +122,8 @@ wget - https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -181,7 +182,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -193,7 +195,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -205,7 +208,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index aec96fe67785..550ac74df4b3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,57 +16,6 @@ // under the License. package com.cloud.kubernetes.cluster; -import java.math.BigInteger; -import java.net.MalformedURLException; -import java.net.URL; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.annotation.AnnotationService; -import org.apache.cloudstack.annotation.dao.AnnotationDao; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiConstants.VMDetails; -import org.apache.cloudstack.api.ResponseObject.ResponseView; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; -import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; -import org.apache.cloudstack.api.response.KubernetesClusterResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.config.ApiServiceConfiguration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; @@ -165,6 +114,53 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.annotation.AnnotationService; +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -428,21 +424,18 @@ private boolean validateServiceOffering(final ServiceOffering serviceOffering, f private void validateDockerRegistryParams(final String dockerRegistryUserName, final String dockerRegistryPassword, - final String dockerRegistryUrl, - final String dockerRegistryEmail) { + final String dockerRegistryUrl) { // if no params related to docker registry specified then nothing to validate so return true if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { + (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty())) { return; } // all params related to docker registry must be specified or nothing if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { + (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()))) { throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); } @@ -451,12 +444,6 @@ private void validateDockerRegistryParams(final String dockerRegistryUserName, } catch (MalformedURLException e) { throw new InvalidParameterValueException("Invalid docker registry url specified"); } - - Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); - Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); - if (!matcher.find()) { - throw new InvalidParameterValueException("Invalid docker registry email specified"); - } } private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { @@ -619,7 +606,6 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); @@ -706,7 +692,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl); Network network = null; if (networkId != null) { @@ -729,7 +715,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu } if (!KubernetesClusterExperimentalFeaturesEnabled.value() && (!Strings.isNullOrEmpty(dockerRegistryUrl) || - !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryEmail) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { + !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { throw new CloudRuntimeException(String.format("Private registry for the Kubernetes cluster is an experimental feature. Use %s configuration for enabling experimental features", KubernetesClusterExperimentalFeaturesEnabled.key())); } } @@ -779,7 +765,6 @@ private void addKubernetesClusterDetails(final KubernetesCluster kubernetesClust final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final boolean networkCleanup = cmd.getNetworkId() == null; Transaction.execute(new TransactionCallbackNoReturn() { @Override @@ -797,9 +782,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); } - if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); - } + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); SecureRandom random = new SecureRandom(); String randomPassword = new BigInteger(130, random).toString(32); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 595aab939994..698ee3176e97 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,26 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; @@ -74,9 +54,9 @@ import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.LaunchPermissionDao; -import com.cloud.storage.VolumeApiService; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.SSHKeyPairVO; @@ -97,6 +77,24 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -164,56 +162,48 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to + + k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + return k8sNodeConfig; + } + + protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { + /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to * use docker private registry */ - String dockerUserName = null; - String dockerPassword = null; - String dockerRegistryUrl = null; - String dockerRegistryEmail = null; + String registryUsername = null; + String registryPassword = null; + String registryUrl = null; + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); for (KubernetesClusterDetailsVO detail : details) { if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - dockerUserName = detail.getValue(); + registryUsername = detail.getValue(); } if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - dockerPassword = detail.getValue(); + registryPassword = detail.getValue(); } if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - dockerRegistryUrl = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { - dockerRegistryEmail = detail.getValue(); + registryUrl = detail.getValue(); } } - if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { - // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section - // optional or conditionally applied - String dockerConfigString = "write_files:\n" + - " - path: /.docker/config.json\n" + - " owner: core:core\n" + - " permissions: '0644'\n" + - " content: |\n" + - " {\n" + - " \"auths\": {\n" + - " {{docker.url}}: {\n" + - " \"auth\": {{docker.secret}},\n" + - " \"email\": {{docker.email}}\n" + - " }\n" + - " }\n" + - " }"; - k8sNodeConfig = k8sNodeConfig.replace("write_files:", dockerConfigString); - final String dockerUrlKey = "{{docker.url}}"; - final String dockerAuthKey = "{{docker.secret}}"; - final String dockerEmailKey = "{{docker.email}}"; - final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; + if (!Strings.isNullOrEmpty(registryUsername) && !Strings.isNullOrEmpty(registryPassword) && !Strings.isNullOrEmpty(registryUrl)) { + // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details + String runCmd = "- bash -x /opt/bin/setup-containerd"; + + String registryEp = registryUrl.split("://")[1]; + k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + + final String usernamePasswordKey = registryUsername + ":" + registryPassword; String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); - k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); } - return k8sNodeConfig; + return k8sConfig; } - protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index b6da75dba81b..c3d59680a9cf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -17,24 +17,6 @@ package com.cloud.kubernetes.cluster.actionworkers; -import java.io.IOException; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.framework.ca.Certificate; -import org.apache.cloudstack.utils.security.CertUtils; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Level; - import com.cloud.dc.DataCenter; import com.cloud.dc.Vlan; import com.cloud.dc.VlanVO; @@ -75,6 +57,23 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; import com.google.common.base.Strings; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -175,6 +174,7 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; } @@ -237,6 +237,7 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 376c148abb7a..d767c2b52bf0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -16,10 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; -import java.security.InvalidParameterException; - -import javax.inject.Inject; - +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -41,10 +41,8 @@ import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; -import com.cloud.kubernetes.cluster.KubernetesClusterService; -import com.cloud.utils.exception.CloudRuntimeException; +import javax.inject.Inject; +import java.security.InvalidParameterException; @APICommand(name = CreateKubernetesClusterCmd.APINAME, description = "Creates a Kubernetes cluster", @@ -140,10 +138,6 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "URL for the docker image private registry") private String dockerRegistryUrl; - @Parameter(name = ApiConstants.DOCKER_REGISTRY_EMAIL, type = CommandType.STRING, - description = "email of the docker image private registry user") - private String dockerRegistryEmail; - @Parameter(name = ApiConstants.NODE_ROOT_DISK_SIZE, type = CommandType.LONG, description = "root disk size in GB for each node") private Long nodeRootDiskSize; @@ -226,10 +220,6 @@ public String getDockerRegistryUrl() { return dockerRegistryUrl; } - public String getDockerRegistryEmail() { - return dockerRegistryEmail; - } - public Long getNodeRootDiskSize() { if (nodeRootDiskSize != null) { if (nodeRootDiskSize < DEFAULT_NODE_ROOT_DISK_SIZE) { diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index db295188eba5..5310e52caf61 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -213,6 +213,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -242,6 +259,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 730477bd4eef..5a4f8b3825d7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -270,6 +270,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -299,6 +316,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index eea369921ac8..e6ffed888f6b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -213,6 +213,23 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root @@ -242,5 +259,6 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 6b6b81f67918..52aa831d75c5 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -226,14 +226,6 @@ }]" :placeholder="apiParams.dockerregistryurl.description"/> - - - -
From 38a156f3346ff975db7ba6afb049f4976849e004 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 17 Jan 2022 13:45:20 +0530 Subject: [PATCH 10/22] Enable updating template type (only) for system owned templates via UI --- ui/src/config/section/image.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 360525397f6e..3bd6d578cd75 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -100,7 +100,6 @@ export default { return (['Admin'].includes(store.userInfo.roletype) || // If admin or owner or belongs to current project (record.domainid === store.userInfo.domainid && record.account === store.userInfo.account) || (record.domainid === store.userInfo.domainid && record.projectid && store.project && store.project.id && record.projectid === store.project.id)) && - record.templatetype !== 'SYSTEM' && record.isready }, popup: true, From 6ea6c072b40c5bfae14d8f1fd93527a0382dc991 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 09:58:46 +0530 Subject: [PATCH 11/22] edit indents --- .../upgrade/SystemVmTemplateRegistration.java | 4 +- ...esClusterResourceModifierActionWorker.java | 1 + .../KubernetesClusterStartWorker.java | 2 + .../resources/conf/k8s-control-node-add.yml | 30 +++++++------- .../main/resources/conf/k8s-control-node.yml | 29 +++++++------- .../src/main/resources/conf/k8s-node.yml | 30 +++++++------- .../util/create-kubernetes-binaries-iso.sh | 2 +- .../debian/opt/cloud/bin/setup/cksnode.sh | 11 ++++-- .../scripts/configure_systemvm_services.sh | 2 +- .../main/java/com/cloud/utils/DigestUtil.java | 39 +++++++++++++++++++ .../java/com/cloud/utils/EncryptionUtil.java | 16 -------- 11 files changed, 98 insertions(+), 68 deletions(-) create mode 100644 utils/src/main/java/com/cloud/utils/DigestUtil.java diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 4119b11ae508..6a927d4d2fec 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,7 +36,7 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; -import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.DigestUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -726,7 +726,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = EncryptionUtil.calculateChecksum(tempFile); + String templateChecksum = DigestUtil.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 698ee3176e97..b624c0770a57 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -164,6 +164,7 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + return k8sNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index c3d59680a9cf..f0aa2a410ec4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -175,6 +175,7 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } @@ -238,6 +239,7 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); + return k8sControlNodeConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 5310e52caf61..877eaf72cb25 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -214,21 +214,21 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 5a4f8b3825d7..02e64093b08a 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -271,21 +271,22 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - echo "Restarting containerd service" - systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index e6ffed888f6b..9ec455e35eb7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -214,21 +214,21 @@ write_files: echo "true" > /home/cloud/success - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index aec95241eb96..ce7626c98c22 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -136,7 +136,7 @@ while read -r line; do sudo ctr image pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` sudo ctr image export "${working_dir}/docker/$image_name.tar" "$line" - sudo ctr image rm "$line" + sudo ctr image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index 240e15f95ff2..169f30ce06a8 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -30,10 +30,15 @@ setup_k8s_node() { sed -i 's/3922/22/g' /etc/ssh/sshd_config # Prevent root login - > /root/.ssh/authorized_keys - passwd -l root +# > /root/.ssh/authorized_keys +# passwd -l root #sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd + # Update containerd configuration + mkdir -p /etc/containerd + containerd config default>/etc/containerd/config.toml + systemctl restart containerd + swapoff -a sudo sed -i '/ swap / s/^/#/' /etc/fstab log_it "Swap disabled" @@ -61,8 +66,6 @@ setup_k8s_node() { log_it "Starting cloud-init services" systemctl enable --now --no-block containerd - systemctl enable --now --no-block docker.socket - systemctl enable --now --no-block docker.service if [ -f /home/cloud/success ]; then systemctl stop cloud-init cloud-config cloud-final systemctl disable cloud-init cloud-config cloud-final diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 8c844b893b53..67bb03491e49 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -19,7 +19,7 @@ set -e set -x -CLOUDSTACK_RELEASE=4.16.0 +CLOUDSTACK_RELEASE=4.16.1 function configure_apache2() { # Enable ssl, rewrite and auth diff --git a/utils/src/main/java/com/cloud/utils/DigestUtil.java b/utils/src/main/java/com/cloud/utils/DigestUtil.java new file mode 100644 index 000000000000..828af672379a --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/DigestUtil.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.utils; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; + +public class DigestUtil { + public static final Logger s_logger = Logger.getLogger(DigestUtil.class.getName()); + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } +} diff --git a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java index 7dbf13b54447..37cea3499d99 100644 --- a/utils/src/main/java/com/cloud/utils/EncryptionUtil.java +++ b/utils/src/main/java/com/cloud/utils/EncryptionUtil.java @@ -20,19 +20,13 @@ import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.jasypt.encryption.pbe.PBEStringEncryptor; import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -74,14 +68,4 @@ public static String generateSignature(String data, String key) { throw new CloudRuntimeException("unable to generate signature", e); } } - - public static String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - s_logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } } From f303f543fa2b1a9f0d86e44e902b87dcc54a08a8 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 15:37:46 +0530 Subject: [PATCH 12/22] Address comments and move cmd from patch file to cloud-init runcmd --- engine/schema/pom.xml | 12 ++---- .../upgrade/SystemVmTemplateRegistration.java | 10 ++++- .../resources/conf/k8s-control-node-add.yml | 1 + .../main/resources/conf/k8s-control-node.yml | 1 + .../src/main/resources/conf/k8s-node.yml | 1 + .../debian/opt/cloud/bin/setup/cksnode.sh | 9 +---- .../main/java/com/cloud/utils/DigestUtil.java | 39 ------------------- .../utils/security/DigestHelper.java | 17 +++++++- 8 files changed, 33 insertions(+), 57 deletions(-) delete mode 100644 utils/src/main/java/com/cloud/utils/DigestUtil.java diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 70556b61fabc..588ed92a0d87 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,8 +122,7 @@ wget - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt + https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -182,8 +181,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -195,8 +193,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -208,8 +205,7 @@ true - - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 6a927d4d2fec..214243886997 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -36,7 +36,6 @@ import com.cloud.upgrade.dao.BasicTemplateDataStoreDaoImpl; import com.cloud.user.Account; import com.cloud.utils.DateUtil; -import com.cloud.utils.DigestUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.GlobalLock; @@ -55,6 +54,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.ini4j.Ini; @@ -679,6 +679,12 @@ public void registerTemplate(Pair hypervisorA } } + /** + * This method parses the metadata file consisting of the systemVM templates information + * @return the version of the systemvm template that is to be used. This is done to in order + * to fallback on the latest available version of the systemVM template when there does not + * exist a template corresponding to the current code version. + */ public static String parseMetadataFile() { try { Ini ini = new Ini(); @@ -726,7 +732,7 @@ private void validateTemplates(Set hypervisorsInUse) } File tempFile = new File(TEMPLATES_PATH + matchedTemplate); - String templateChecksum = DigestUtil.calculateChecksum(tempFile); + String templateChecksum = DigestHelper.calculateChecksum(tempFile); if (!templateChecksum.equals(NewTemplateChecksum.get(getHypervisorType(hypervisor)))) { LOGGER.error(String.format("Checksum mismatch: %s != %s ", templateChecksum, NewTemplateChecksum.get(getHypervisorType(hypervisor)))); templatesFound = false; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 877eaf72cb25..0ca165fa6bee 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -260,6 +260,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 02e64093b08a..5087ec262dde 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -318,6 +318,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 9ec455e35eb7..c2cc52133bfc 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -260,5 +260,6 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh index 169f30ce06a8..ac672a265399 100755 --- a/systemvm/debian/opt/cloud/bin/setup/cksnode.sh +++ b/systemvm/debian/opt/cloud/bin/setup/cksnode.sh @@ -30,15 +30,10 @@ setup_k8s_node() { sed -i 's/3922/22/g' /etc/ssh/sshd_config # Prevent root login -# > /root/.ssh/authorized_keys -# passwd -l root + > /root/.ssh/authorized_keys + passwd -l root #sed -i 's#root:x:0:0:root:/root:/bin/bash#root:x:0:0:root:/root:/sbin/nologin#' /etc/passwd - # Update containerd configuration - mkdir -p /etc/containerd - containerd config default>/etc/containerd/config.toml - systemctl restart containerd - swapoff -a sudo sed -i '/ swap / s/^/#/' /etc/fstab log_it "Swap disabled" diff --git a/utils/src/main/java/com/cloud/utils/DigestUtil.java b/utils/src/main/java/com/cloud/utils/DigestUtil.java deleted file mode 100644 index 828af672379a..000000000000 --- a/utils/src/main/java/com/cloud/utils/DigestUtil.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.utils; - -import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; - -public class DigestUtil { - public static final Logger s_logger = Logger.getLogger(DigestUtil.class.getName()); - public static String calculateChecksum(File file) { - try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { - return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); - } catch (IOException e) { - String errMsg = "Failed to calculate template checksum"; - s_logger.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - } -} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java index f856b1f2892d..a8307cc751fe 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/DigestHelper.java @@ -16,18 +16,23 @@ // under the License. package org.apache.cloudstack.utils.security; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map; public class DigestHelper { - + public static final Logger s_logger = Logger.getLogger(DigestHelper.class.getName()); public static ChecksumValue digest(String algorithm, InputStream is) throws NoSuchAlgorithmException, IOException { MessageDigest digest = MessageDigest.getInstance(algorithm); ChecksumValue checksum = null; @@ -131,4 +136,14 @@ protected static boolean isAlgorithmPresent(String checksum) { public static String getHashValueFromChecksumValue(String checksum) { return isAlgorithmPresent(checksum) ? new ChecksumValue(checksum).getChecksum() : checksum; } + + public static String calculateChecksum(File file) { + try (InputStream is = Files.newInputStream(Paths.get(file.getPath()))) { + return org.apache.commons.codec.digest.DigestUtils.md5Hex(is); + } catch (IOException e) { + String errMsg = "Failed to calculate template checksum"; + s_logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } } From 53a28d174427d65bbc0db0903f1fdf3cb5824836 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 18 Jan 2022 17:00:19 +0530 Subject: [PATCH 13/22] temporary change --- engine/schema/pom.xml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 588ed92a0d87..70556b61fabc 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,7 +122,8 @@ wget - https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -181,7 +182,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -193,7 +195,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -205,7 +208,8 @@ true - https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + + http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} From a762d8c40fc1052e35a90cb0f74ad1f1dc336951 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 21 Jan 2022 12:08:11 +0530 Subject: [PATCH 14/22] update k8s test to use k8s version 1.21.5 (instead of 1.21.3 - due to https://github.com/kubernetes/kubernetes/pull/104530) --- .../upgrade/SystemVmTemplateRegistration.java | 5 +---- ...esClusterResourceModifierActionWorker.java | 4 ++++ .../KubernetesClusterStartWorker.java | 7 +++++++ .../smoke/test_kubernetes_clusters.py | 20 +++++++++---------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 214243886997..78afe3a45c02 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -470,7 +470,7 @@ private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { template.setCrossZones(true); template.setHypervisorType(details.getHypervisorType()); template.setState(VirtualMachineTemplate.State.Inactive); - template.setDeployAsIs(Hypervisor.HypervisorType.VMware.equals(details.getHypervisorType())); + template.setDeployAsIs(false); template = vmTemplateDao.persist(template); return template; } @@ -828,9 +828,6 @@ private void updateRegisteredTemplateDetails(Long templateId, Map.Entry hypervisorAndTemplateName) { templateVO.setUrl(NewTemplateUrl.get(hypervisorAndTemplateName.getKey())); templateVO.setChecksum(NewTemplateChecksum.get(hypervisorAndTemplateName.getKey())); - if (Hypervisor.HypervisorType.VMware == templateVO.getHypervisorType()) { - templateVO.setDeployAsIs(true); - } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", hypervisorAndTemplateName.getKey().name()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index b624c0770a57..a2525ba41819 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -75,6 +75,7 @@ import com.cloud.vm.Nic; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; import org.apache.cloudstack.api.ApiConstants; @@ -360,6 +361,9 @@ protected UserVm createKubernetesNode(String joinIp) throws ManagementServerExce if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sNodeConfig = null; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index f0aa2a410ec4..a728182c53de 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -56,6 +56,7 @@ import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import com.google.common.base.Strings; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.framework.ca.Certificate; @@ -198,6 +199,9 @@ private UserVm createKubernetesControlNode(final Network network, String serverI if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); boolean haSupported = isKubernetesVersionSupportsHA(); @@ -256,6 +260,9 @@ private UserVm createKubernetesAdditionalControlNode(final String joinIp, final if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + if (Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())) { + customParameterMap.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "scsi"); + } String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sControlNodeConfig = null; diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 0387ddf9cc1b..46f1f5d02bdc 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -101,12 +101,12 @@ def setUpClass(cls): (cls.services["cks_kubernetes_versions"]["1.20.9"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.20.9"]["url"], e)) if cls.setup_failed == False: try: - cls.kubernetes_version_1_21_3 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.3"]) - cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_3.id) + cls.kubernetes_version_1_21_5 = cls.addKubernetesSupportedVersion(cls.services["cks_kubernetes_versions"]["1.21.5"]) + cls.kubernetes_version_ids.append(cls.kubernetes_version_1_21_5.id) except Exception as e: cls.setup_failed = True cls.debug("Failed to get Kubernetes version ISO in ready state, version=%s, url=%s, %s" % - (cls.services["cks_kubernetes_versions"]["1.21.3"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.3"]["url"], e)) + (cls.services["cks_kubernetes_versions"]["1.21.5"]["semanticversion"], cls.services["cks_kubernetes_versions"]["1.21.5"]["url"], e)) if cls.setup_failed == False: cks_offering_data = cls.services["cks_service_offering"] @@ -349,7 +349,7 @@ def test_01_invalid_upgrade_kubernetes_cluster(self): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Downgrading Kubernetes cluster with ID: %s to a lower version. This should fail!" % k8s_cluster.id) @@ -362,7 +362,7 @@ def test_01_invalid_upgrade_kubernetes_cluster(self): self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -381,12 +381,12 @@ def test_02_upgrade_kubernetes_cluster(self): time.sleep(self.services["sleep"]) self.debug("Upgrading Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) return @attr(tags=["advanced", "smoke"], required_hardware="true") @@ -434,7 +434,7 @@ def test_04_autoscale_kubernetes_cluster(self): if self.setup_failed == True: self.fail("Setup incomplete") global k8s_cluster - k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_3) + k8s_cluster = self.getValidKubernetesCluster(version=self.kubernetes_version_1_21_5) self.debug("Autoscaling Kubernetes cluster with ID: %s" % k8s_cluster.id) try: @@ -535,12 +535,12 @@ def test_08_upgrade_kubernetes_ha_cluster(self): self.debug("Upgrading HA Kubernetes cluster with ID: %s" % k8s_cluster.id) try: - k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_3.id) + k8s_cluster = self.upgradeKubernetesCluster(k8s_cluster.id, self.kubernetes_version_1_21_5.id) except Exception as e: self.deleteKubernetesClusterAndVerify(k8s_cluster.id, False, True) self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_3.id) + self.verifyKubernetesClusterUpgrade(k8s_cluster, self.kubernetes_version_1_21_5.id) self.debug("Kubernetes cluster with ID: %s successfully upgraded" % k8s_cluster.id) return From e16c64ba6e6660731e34ac761903ec7a3b916967 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 2 Feb 2022 12:21:16 +0530 Subject: [PATCH 15/22] smooth upgrade of cks clusters --- .../java/com/cloud/vm/VmDetailConstants.java | 1 + .../META-INF/db/schema-41600to41610.sql | 21 +++++++ .../KubernetesClusterActionWorker.java | 57 +++++++++++++++++-- ...esClusterResourceModifierActionWorker.java | 7 ++- .../KubernetesClusterScaleWorker.java | 8 ++- .../KubernetesClusterStartWorker.java | 11 ++-- .../KubernetesClusterUpgradeWorker.java | 10 ++-- 7 files changed, 94 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index bf43940c0a02..5fbbc872d341 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -73,4 +73,5 @@ public interface VmDetailConstants { String DISK_OFFERING = "diskOffering"; String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; + String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index 3967c69954f3..eac0729f3bdb 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -129,3 +129,24 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_vpn_user', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)'); UPDATE `cloud`.`vm_template` set deploy_as_is = 0 where id = 8; + +CREATE PROCEDURE `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`() +BEGIN + DECLARE vmid BIGINT +; DECLARE done TINYINT DEFAULT FALSE +; DECLARE vmidcursor CURSOR FOR SELECT DISTINCT(vm_id) FROM `cloud`.`kubernetes_cluster_vm_map` +; DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE +; OPEN vmidcursor +; vmid_loop:LOOP + FETCH NEXT FROM vmidcursor INTO vmid +; IF done THEN + LEAVE vmid_loop +; ELSE + INSERT `cloud`.`user_vm_details` (vm_id, name, value, display) VALUES (vmid, 'controlNodeLoginUser', 'core', 1) +; END IF +; END LOOP +; CLOSE vmidcursor +; END; + +CALL `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`(); +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index cf0403b4000a..60111d2a397b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -57,9 +57,13 @@ import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.UserVmDetailsDao; import com.google.common.base.Strings; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.ca.CAManager; @@ -78,8 +82,11 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; public class KubernetesClusterActionWorker { @@ -118,6 +125,8 @@ public class KubernetesClusterActionWorker { @Inject protected UserVmDao userVmDao; @Inject + protected UserVmDetailsDao userVmDetailsDao; + @Inject protected UserVmService userVmService; @Inject protected VlanDao vlanDao; @@ -173,6 +182,27 @@ protected String readResourceFile(String resource) throws IOException { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), StringUtils.getPreferredCharset()); } + protected String getControlNodeLoginUser() { + List vmMapVOList = getKubernetesClusterVMMaps(); + if (vmMapVOList.size() > 0) { + long vmId = vmMapVOList.get(0).getVmId(); + UserVmVO userVM = userVmDao.findById(vmId); + if (userVM == null) { + throw new CloudRuntimeException("Failed to find login user, Unable to log in to node to fetch details"); + } + Set vm = new HashSet<>(); + vm.add(userVM.getName()); + UserVmDetailVO vmDetail = userVmDetailsDao.findDetail(vmId, VmDetailConstants.CKS_CONTROL_NODE_LOGIN_USER); + if (vmDetail != null && !org.apache.commons.lang3.StringUtils.isEmpty(vmDetail.getValue())) { + return vmDetail.getValue(); + } else { + return CLUSTER_NODE_VM_USER; + } + } else { + return CLUSTER_NODE_VM_USER; + } + } + protected void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.INFO) { if (LOGGER.isInfoEnabled()) { @@ -408,6 +438,20 @@ protected List getKubernetesClusterVMs() { return vmList; } + protected void updateLoginUserDetails(List clusterVMs) { + if (clusterVMs == null) { + clusterVMs = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + } + if (!CollectionUtils.isEmpty(clusterVMs)) { + for (Long vmId : clusterVMs) { + UserVm controlNode = userVmDao.findById(vmId); + if (controlNode != null) { + userVmDetailsDao.addDetail(vmId, VmDetailConstants.CKS_CONTROL_NODE_LOGIN_USER, CLUSTER_NODE_VM_USER, true); + } + } + } + } + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); try { @@ -422,13 +466,14 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve protected boolean createCloudStackSecret(String[] keys) { File pkFile = getManagementServerSshPublicKeyFile(); Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + List vmMapVOList = getKubernetesClusterVMMaps(); publicIpAddress = publicIpSshPort.first(); sshPort = publicIpSshPort.second(); try { final String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { @@ -467,10 +512,10 @@ protected void copyScripts(String nodeAddress, final int sshPort) { protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String desitnation) { try { - SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", file.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, desitnation); - SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); @@ -494,7 +539,7 @@ protected boolean taintControlNodes() { publicIpAddress = publicIpSshPort.first(); sshPort = publicIpSshPort.second(); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, commands.toString(), 10000, 10000, 60000); return result.first(); } catch (Exception e) { @@ -519,7 +564,7 @@ protected boolean deployProvider() { try { String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it @@ -534,7 +579,7 @@ protected boolean deployProvider() { } // If at first you don't succeed ... - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index a2525ba41819..6783de3d0760 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -611,7 +611,7 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { if (enable) { String command = String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", kubernetesCluster.getUuid(), maxSize, minSize); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it @@ -626,7 +626,7 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { } // If at first you don't succeed ... - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); @@ -634,7 +634,7 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { } updateKubernetesClusterEntry(true, minSize, maxSize); } else { - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/autoscale-kube-cluster -d"), 10000, 10000, 60000); if (!result.first()) { @@ -649,6 +649,7 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { return false; } finally { // Deploying the autoscaler might fail but it can be deployed manually too, so no need to go to an alert state + updateLoginUserDetails(null); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 0c2c17d3aa4c..aa9606d473f0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -26,6 +26,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; @@ -192,13 +193,13 @@ private boolean removeKubernetesClusterNode(final String ipAddress, final int po while (retryCounter < retries) { retryCounter++; try { - Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + Pair result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); if (!result.first()) { LOGGER.warn(String.format("Draining node: %s on VM : %s in Kubernetes cluster : %s unsuccessful", hostName, userVm.getDisplayName(), kubernetesCluster.getName())); } else { - result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + result = SshHelper.sshExecute(ipAddress, port, getControlNodeLoginUser(), pkFile, null, String.format("sudo /opt/bin/kubectl delete node %s", hostName), 10000, 10000, 30000); if (result.first()) { @@ -359,6 +360,7 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun launchPermissionDao.persist(launchPermission); try { clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); + updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e); } @@ -372,7 +374,7 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, scaleTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); deleteTemplateLaunchPermission(); if (!readyNodesCountValid) { // Scaling failed diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a728182c53de..f9d42f617568 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -59,6 +59,7 @@ import com.cloud.vm.VmDetailConstants; import com.google.common.base.Strings; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.commons.codec.binary.Base64; @@ -75,6 +76,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -465,7 +467,7 @@ private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { return true; } } - String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime); + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime); if (!Strings.isNullOrEmpty(kubeConfig)) { final String controlVMPrivateIpAddress = getControlVmPrivateIp(); if (!Strings.isNullOrEmpty(controlVMPrivateIpAddress)) { @@ -485,7 +487,7 @@ private boolean isKubernetesClusterDashboardServiceRunning(final boolean onCreat return true; } } - if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime, 15000)) { + if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, timeoutTime, 15000)) { kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(true), false); return true; } @@ -581,7 +583,7 @@ public boolean startKubernetesClusterOnCreate() { sshPort = publicIpSshPort.second(); updateKubernetesClusterEntryEndpoint(); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, startTimeoutTime, 15000); + getControlNodeLoginUser(), sshKeyFile, startTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s as it does not have desired number of nodes in ready state", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); @@ -594,6 +596,7 @@ public boolean startKubernetesClusterOnCreate() { } taintControlNodes(); deployProvider(); + updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -648,7 +651,7 @@ public boolean reconcileAlertCluster() { } long actualNodeCount = 0; try { - actualNodeCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile); + actualNodeCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile); } catch (Exception e) { return false; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ad289bf81a95..6e22c9f336a7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -63,7 +63,7 @@ protected void retrieveScriptFiles() { private Pair runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; - SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo ./%s %s %s %s %s", upgradeScriptFile.getName(), @@ -71,7 +71,7 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in index == 0 ? "true" : "false", KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false", Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())); - return SshHelper.sshExecute(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); } @@ -90,7 +90,7 @@ private void upgradeKubernetesClusterNodes() { vm.getDisplayName(), kubernetesCluster.getName(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); } try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, String.format("sudo /opt/bin/kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); } catch (Exception e) { @@ -114,11 +114,11 @@ private void upgradeKubernetesClusterNodes() { if (System.currentTimeMillis() > upgradeTimeoutTime) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } - if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { + if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to uncordon Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (i == 0) { // Wait for control node to get in Ready state - if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } From 1e29f69ecbe8d0498a2aceece51e68dc132226ae Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 7 Feb 2022 17:07:35 +0530 Subject: [PATCH 16/22] update pom file with temp download.cloudstack.org testing links --- engine/schema/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 70556b61fabc..0081570109c9 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -123,7 +123,7 @@ - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/md5sum.txt + https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -183,7 +183,7 @@ true - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -196,7 +196,7 @@ true - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -209,7 +209,7 @@ true - http://10.0.3.122/systemvmtemplate/custom/cks-debian/${cs.version}/1/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} From 16e682644f565da061a596cfc1396789973ddc59 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 8 Feb 2022 14:54:33 +0530 Subject: [PATCH 17/22] fix pom --- engine/schema/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 0081570109c9..5a656fbb0aba 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -183,7 +183,7 @@ true - https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -196,7 +196,7 @@ true - https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -209,7 +209,7 @@ true - https://download.cloudstack.org/testing/systemvmtemplate/{cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum} From 2875fddcbf5f3ca56a4a37c6ecac42b3b771d154 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 9 Feb 2022 17:55:57 +0530 Subject: [PATCH 18/22] add cgroup config for containerd --- .../src/main/resources/conf/k8s-control-node-add.yml | 1 + .../src/main/resources/conf/k8s-control-node.yml | 1 + .../kubernetes-service/src/main/resources/conf/k8s-node.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 0ca165fa6bee..82fb75d3777f 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -260,6 +260,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 5087ec262dde..b7b71af6a980 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -318,6 +318,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index c2cc52133bfc..0639a07f0710 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -260,6 +260,7 @@ write_files: runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml + - sed -i '/\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\]/a '"\\ SystemdCgroup=true"'' /etc/containerd/config.toml - systemctl restart containerd - [ systemctl, start, setup-kube-system ] - [ systemctl, start, deploy-kube-system ] From b6dc3736c1ac645abd53e7b77226980be26b537f Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 10 Feb 2022 18:05:15 +0530 Subject: [PATCH 19/22] add systemd config for kubelet --- .../src/main/resources/conf/k8s-control-node-add.yml | 2 ++ .../src/main/resources/conf/k8s-control-node.yml | 2 ++ .../kubernetes-service/src/main/resources/conf/k8s-node.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 82fb75d3777f..b403afbf200e 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -107,6 +107,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index b7b71af6a980..7002c74ab87c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -127,6 +127,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 0639a07f0710..9f9cb2e0a71c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -107,6 +107,8 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do From 1accc9f7560c80be03eadd1c13800a4017bcf710 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 11 Feb 2022 18:14:34 +0530 Subject: [PATCH 20/22] add additional info during image registry config --- .../KubernetesClusterResourceModifierActionWorker.java | 4 ++++ .../src/main/resources/conf/k8s-control-node-add.yml | 2 +- .../src/main/resources/conf/k8s-control-node.yml | 2 +- .../kubernetes-service/src/main/resources/conf/k8s-node.yml | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 6783de3d0760..67fb093534b3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -197,11 +197,15 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { final String registryUrlKey = "{{registry.url}}"; final String registryUrlEpKey = "{{registry.url.endpoint}}"; final String registryAuthKey = "{{registry.token}}"; + final String registryUname = "{{registry.username}}"; + final String registryPsswd = "{{registry.password}}"; final String usernamePasswordKey = registryUsername + ":" + registryPassword; String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryUname, registryUsername); + k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); } return k8sConfig; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index b403afbf200e..3156f378674e 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -222,7 +222,7 @@ write_files: #!/bin/bash -e export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" echo "creating config file for containerd" containerd config default > /etc/containerd/config.toml diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 7002c74ab87c..4dcaf4960723 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -279,7 +279,7 @@ write_files: #!/bin/bash -e export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" echo "creating config file for containerd" containerd config default > /etc/containerd/config.toml diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 9f9cb2e0a71c..eedc1d0c8c88 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -222,7 +222,7 @@ write_files: #!/bin/bash -e export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n \tauth = \"\" \n \tidentitytoken = \"{{registry.token}}\"" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" echo "creating config file for containerd" containerd config default > /etc/containerd/config.toml From a725d311a94884add91442770307d1429da76af6 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 15 Feb 2022 14:26:30 +0530 Subject: [PATCH 21/22] Update engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql Co-authored-by: Suresh Kumar Anaparti --- .../src/main/resources/META-INF/db/schema-41600to41610.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql index 05c343dc7e8e..d53e2181cef8 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41600to41610.sql @@ -128,7 +128,7 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id -- Add PK to cloud_usage.usage_vpn_user CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_vpn_user', 'id', 'BIGINT(20) NOT NULL AUTO_INCREMENT FIRST, ADD PRIMARY KEY (`id`)'); -UPDATE `cloud`.`vm_template` set deploy_as_is = 0 where id = 8; +UPDATE `cloud`.`vm_template` SET deploy_as_is = 0 WHERE id = 8; CREATE PROCEDURE `cloud`.`UPDATE_KUBERNETES_NODE_DETAILS`() BEGIN From a75fa7f4bb72d812d38aec2544f758864bab0afe Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 15 Feb 2022 15:06:59 +0530 Subject: [PATCH 22/22] update to official links --- engine/schema/pom.xml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 5a656fbb0aba..588ed92a0d87 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -122,8 +122,7 @@ wget - - https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/md5sum.txt + https://download.cloudstack.org/systemvm/${cs.version}/md5sum.txt ${basedir}/dist/systemvm-templates/ true true @@ -182,8 +181,7 @@ true - - https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-kvm.qcow2.bz2 ${basedir}/dist/systemvm-templates/ ${kvm.checksum} @@ -195,8 +193,7 @@ true - - https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-vmware.ova ${basedir}/dist/systemvm-templates/ ${vmware.checksum} @@ -208,8 +205,7 @@ true - - https://download.cloudstack.org/testing/systemvmtemplate/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 + https://download.cloudstack.org/systemvm/${cs.version}/systemvmtemplate-${cs.version}.${patch.version}-xen.vhd.bz2 ${basedir}/dist/systemvm-templates/ ${xen.checksum}