From 40e3338001b2f1c239533bedac5b60d8b9d2f631 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 27 May 2022 13:48:49 +0530 Subject: [PATCH 1/3] cks: Fix when deployed on a nw without internet access --- ...esClusterResourceModifierActionWorker.java | 50 +-- .../KubernetesClusterStartWorker.java | 42 +-- .../resources/conf/k8s-control-node-add.yml | 272 --------------- .../main/resources/conf/k8s-control-node.yml | 329 ------------------ .../src/main/resources/conf/k8s-node.yml | 289 ++++++--------- scripts/cks/autoscale-kube-cluster | 93 +++++ scripts/cks/deploy-cloudstack-secret | 68 ++++ scripts/cks/deploy-kube-system | 104 ++++++ scripts/cks/deploy-provider | 36 ++ scripts/cks/mount-cks-iso | 41 +++ scripts/cks/setup-containerd-registry | 18 + scripts/cks/setup-kube-system | 92 +++++ scripts/cks/upgrade-kubernetes.sh | 143 ++++++++ .../util/create-kubernetes-binaries-iso.sh | 11 + 14 files changed, 750 insertions(+), 838 deletions(-) delete mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml delete mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml create mode 100755 scripts/cks/autoscale-kube-cluster create mode 100755 scripts/cks/deploy-cloudstack-secret create mode 100755 scripts/cks/deploy-kube-system create mode 100755 scripts/cks/deploy-provider create mode 100755 scripts/cks/mount-cks-iso create mode 100755 scripts/cks/setup-containerd-registry create mode 100755 scripts/cks/setup-kube-system create mode 100755 scripts/cks/upgrade-kubernetes.sh diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 81db87a7c7de..187c87330501 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -135,6 +135,22 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected VolumeDao volumeDao; protected String kubernetesClusterNodeNamePrefix; + protected static final String apiServerCertPlaceholder = "{{ k8s_control_node.apiserver.crt }}"; + protected static final String apiServerKeyPlaceholder = "{{ k8s_control_node.apiserver.key }}"; + protected static final String caCertPlaceholder = "{{ k8s_control_node.ca.crt }}"; + protected static final String sshPubKeyPlaceholder = "{{ k8s.ssh.pub.key }}"; + protected static final String clusterTokenPlaceholder = "{{ k8s_control_node.cluster.token }}"; + protected static final String clusterInitArgsPlaceholder = "{{ k8s_control_node.cluster.initargs }}"; + protected static final String ejectIsoPlaceholder = "{{ k8s.eject.iso }}"; + protected static final String joinIpPlaceholder = "{{ k8s_control_node.join_ip }}"; + protected static final String nodeTypePlaceholder = "{{ k8s.node.type }}"; + protected static final String clusterHACertificateKeyPlaceholder = "{{ k8s_control_node.cluster.ha.certificate.key }}"; + protected static final String registryUrlPlaceholder = "{{ k8s.registry.url }}"; + protected static final String registryUrlEndpointPlaceholder = "{{ k8s.registry.url.endpoint }}"; + protected static final String registryUsernamePlaceholder = "{{ k8s.registry.username }}"; + protected static final String registryPasswordPlaceholder = "{{ k8s.registry.password }}"; + protected static final String registryTokenPlaceholder = "{{ k8s.registry.token }}"; + protected static final String cksUserdataFile = "/conf/k8s-node.yml"; protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -146,11 +162,7 @@ protected void init() { } private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String joinIpKey = "{{ k8s_control_node.join_ip }}"; - final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; - final String ejectIsoKey = "{{ k8s.eject.iso }}"; + String k8sNodeConfig = readResourceFile(cksUserdataFile); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -159,10 +171,11 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); - k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sNodeConfig = k8sNodeConfig.replace(nodeTypePlaceholder, "worker"); + k8sNodeConfig = k8sNodeConfig.replace(sshPubKeyPlaceholder, pubKey); + k8sNodeConfig = k8sNodeConfig.replace(joinIpPlaceholder, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sNodeConfig = k8sNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); @@ -191,23 +204,18 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details - String runCmd = "- bash -x /opt/bin/setup-containerd"; + String runCmd = "- bash -x /opt/bin/setup-containerd-registry"; - String registryEp = registryUrl.split("://")[1]; + String registryEndpoint = registryUrl.split("://")[1]; k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); - final String registryUrlKey = "{{registry.url}}"; - final String registryUrlEpKey = "{{registry.url.endpoint}}"; - final String registryAuthKey = "{{registry.token}}"; - final String registryUname = "{{registry.username}}"; - final String registryPsswd = "{{registry.password}}"; final String usernamePasswordKey = registryUsername + ":" + registryPassword; String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); - k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); - k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); - k8sConfig = k8sConfig.replace(registryUname, registryUsername); - k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); - k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); + k8sConfig = k8sConfig.replace(registryUrlPlaceholder, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEndpointPlaceholder, registryEndpoint); + k8sConfig = k8sConfig.replace(registryUsernamePlaceholder, registryUsername); + k8sConfig = k8sConfig.replace(registryPasswordPlaceholder, registryPassword); + k8sConfig = k8sConfig.replace(registryTokenPlaceholder, base64Auth); } return k8sConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 6612a7608325..2c8e9b9235bc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -133,14 +133,7 @@ private boolean isKubernetesVersionSupportsHA() { private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, final String hostName, final boolean haSupported, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); - final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; - final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; - final String caCert = "{{ k8s_control_node.ca.crt }}"; - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterToken = "{{ k8s_control_node.cluster.token }}"; - final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}"; - final String ejectIsoKey = "{{ k8s.eject.iso }}"; + String k8sControlNodeConfig = readResourceFile(cksUserdataFile); final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); if (!serverIp.equals(controlNodeIp)) { @@ -152,9 +145,9 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); - k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); - k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); - k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCertPlaceholder, tlsClientCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKeyPlaceholder, tlsPrivateKey.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(caCertPlaceholder, tlsCaCert.replace("\n", "\n ")); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -163,8 +156,9 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(nodeTypePlaceholder, "control-plane"); + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKeyPlaceholder, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", @@ -174,8 +168,8 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); - k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsPlaceholder, initArgs); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; @@ -237,12 +231,7 @@ private UserVm createKubernetesControlNode(final Network network, String serverI } private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); - final String joinIpKey = "{{ k8s_control_node.join_ip }}"; - final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}"; - final String ejectIsoKey = "{{ k8s.eject.iso }}"; + String k8sControlNodeConfig = readResourceFile(cksUserdataFile); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -251,11 +240,12 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); - k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); - k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(nodeTypePlaceholder, "control-plane-add"); + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKeyPlaceholder, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpPlaceholder, joinIp); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKeyPlaceholder, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml deleted file mode 100644 index 335ae9194e6d..000000000000 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ /dev/null @@ -1,272 +0,0 @@ -#cloud-config -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -users: - - name: cloud - sudo: ALL=(ALL) NOPASSWD:ALL - shell: /bin/bash - ssh_authorized_keys: - {{ k8s.ssh.pub.key }} - -write_files: - - path: /opt/bin/setup-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - - ISO_MOUNT_DIR=/mnt/k8sdisk - BINARIES_DIR=${ISO_MOUNT_DIR}/ - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - ATTEMPT_ONLINE_INSTALL=false - setup_complete=false - - OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=100 - offline_attempts=1 - MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 - EJECT_ISO_FROM_OS={{ k8s.eject.iso }} - crucial_cmd_attempts=1 - iso_drive_path="" - while true; do - if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then - echo "Warning: Offline install timed out!" - break - fi - set +e - output=`blkid -o device -t TYPE=iso9660` - set -e - if [ "$output" != "" ]; then - while read -r line; do - if [ ! -d "${ISO_MOUNT_DIR}" ]; then - mkdir "${ISO_MOUNT_DIR}" - fi - retval=0 - set +e - mount -o ro "${line}" "${ISO_MOUNT_DIR}" - retval=$? - set -e - if [ $retval -eq 0 ]; then - if [ -d "$BINARIES_DIR" ]; then - iso_drive_path="${line}" - break - else - umount "${line}" && rmdir "${ISO_MOUNT_DIR}" - fi - fi - done <<< "$output" - fi - if [ -d "$BINARIES_DIR" ]; then - break - fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" - sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP - offline_attempts=$[$offline_attempts + 1] - done - - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - - if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then - eject "${iso_drive_path}" - fi - fi - if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - fi - - systemctl enable kubelet && systemctl start kubelet - modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - - if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - fi - - - path: /opt/bin/deploy-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - - if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 - fi - modprobe ip_vs - modprobe ip_vs_wrr - modprobe ip_vs_sh - modprobe nf_conntrack - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification - - sudo touch /home/cloud/success - echo "true" > /home/cloud/success - - - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd - - - path: /etc/systemd/system/setup-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - - [Service] - Type=simple - StartLimitInterval=0 - Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version - ExecStart=/opt/bin/deploy-kube-system - -runcmd: - - chown -R cloud:cloud /home/cloud/.ssh - - containerd config default > /etc/containerd/config.toml - - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - systemctl daemon-reload - - systemctl restart containerd - - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] - diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml deleted file mode 100644 index 76adc5a7d296..000000000000 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ /dev/null @@ -1,329 +0,0 @@ -#cloud-config -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - ---- -users: - - name: cloud - sudo: ALL=(ALL) NOPASSWD:ALL - shell: /bin/bash - ssh_authorized_keys: - {{ k8s.ssh.pub.key }} - -write_files: - - path: /etc/conf.d/nfs - permissions: '0644' - content: | - OPTS_RPC_MOUNTD="" - - - path: /etc/kubernetes/pki/cloudstack/ca.crt - permissions: '0644' - content: | - {{ k8s_control_node.ca.crt }} - - - path: /etc/kubernetes/pki/cloudstack/apiserver.crt - permissions: '0644' - content: | - {{ k8s_control_node.apiserver.crt }} - - - path: /etc/kubernetes/pki/cloudstack/apiserver.key - permissions: '0600' - content: | - {{ k8s_control_node.apiserver.key }} - - - path: /opt/bin/setup-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - - ISO_MOUNT_DIR=/mnt/k8sdisk - BINARIES_DIR=${ISO_MOUNT_DIR}/ - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - ATTEMPT_ONLINE_INSTALL=false - setup_complete=false - - OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=100 - offline_attempts=1 - MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 - EJECT_ISO_FROM_OS={{ k8s.eject.iso }} - crucial_cmd_attempts=1 - iso_drive_path="" - while true; do - if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then - echo "Warning: Offline install timed out!" - break - fi - set +e - output=`blkid -o device -t TYPE=iso9660` - set -e - if [ "$output" != "" ]; then - while read -r line; do - if [ ! -d "${ISO_MOUNT_DIR}" ]; then - mkdir "${ISO_MOUNT_DIR}" - fi - retval=0 - set +e - mount -o ro "${line}" "${ISO_MOUNT_DIR}" - retval=$? - set -e - if [ $retval -eq 0 ]; then - if [ -d "$BINARIES_DIR" ]; then - iso_drive_path="${line}" - break - else - umount "${line}" && rmdir "${ISO_MOUNT_DIR}" - fi - fi - done <<< "$output" - fi - if [ -d "$BINARIES_DIR" ]; then - break - fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" - sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP - offline_attempts=$[$offline_attempts + 1] - done - - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - - if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then - eject "${iso_drive_path}" - fi - fi - if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - fi - - systemctl enable kubelet && systemctl start kubelet - modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - - if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - fi - - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm init failed!" - exit 1 - fi - retval=0 - set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - - - path: /opt/bin/deploy-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - - if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 - fi - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - export KUBECONFIG=/etc/kubernetes/admin.conf - - mkdir -p /root/.kube - cp -i /etc/kubernetes/admin.conf /root/.kube/config - chown $(id -u):$(id -g) /root/.kube/config - echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - - if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then - ### Network, dashboard configs available offline ### - echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml - rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - else - /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" - /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml - fi - - /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true - /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true - /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - - sudo touch /home/cloud/success - echo "true" > /home/cloud/success - - - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd - - - path: /etc/systemd/system/setup-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - - [Service] - Type=simple - StartLimitInterval=0 - Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version - ExecStart=/opt/bin/deploy-kube-system - -runcmd: - - chown -R cloud:cloud /home/cloud/.ssh - - containerd config default > /etc/containerd/config.toml - - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - systemctl daemon-reload - - systemctl restart containerd - - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] - diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 86966245c83e..6834d40874f6 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -25,35 +25,98 @@ users: {{ k8s.ssh.pub.key }} write_files: - - path: /opt/bin/setup-kube-system - permissions: '0700' + + # Control plane specific + - path: /etc/conf.d/nfs + permissions: '0644' + content: | + OPTS_RPC_MOUNTD="" + + - path: /etc/kubernetes/pki/cloudstack/apiserver.crt + permissions: '0644' + content: | + {{ k8s_control_node.apiserver.crt }} + + - path: /etc/kubernetes/pki/cloudstack/apiserver.key + permissions: '0600' + content: | + {{ k8s_control_node.apiserver.key }} + + - path: /etc/kubernetes/pki/cloudstack/ca.crt + permissions: '0644' + content: | + {{ k8s_control_node.ca.crt }} + + # Kubeadm extra args + - path: /opt/cks/cluster_token + permissions: '0644' + content: | + {{ k8s_control_node.cluster.token }} + + - path: /opt/cks/cluster_init_args + permissions: '0644' + content: | + {{ k8s_control_node.cluster.initargs }} + + - path: /opt/cks/cluster_join_ip + permissions: '0644' + content: | + {{ k8s_control_node.join_ip }} + + - path: /opt/cks/cluster_ha_cert_key + permissions: '0644' + content: | + {{ k8s_control_node.cluster.ha.certificate.key }} + + # Custom registry + - path: /opt/cks/registry_url + permissions: '0644' + content: | + {{ k8s.registry.url }} + + - path: /opt/cks/registry_url_endpoint + permissions: '0644' + content: | + {{ k8s.registry.url.endpoint }} + + - path: /opt/cks/registry_username + permissions: '0644' + content: | + {{ k8s.registry.username }} + + - path: /opt/cks/registry_password + permissions: '0644' + content: | + {{ k8s.registry.password }} + + - path: /opt/cks/registry_token + permissions: '0644' + content: | + {{ k8s.registry.token }} + + # Misc settings + - path: /opt/cks/node_type + permissions: '0644' + content: | + {{ k8s.node.type }} + + - path: /opt/cks/eject_iso + permissions: '0644' + content: | + {{ k8s.eject.iso }} + + # To get things started + - path: /opt/bin/mount_cks_iso + permissions: '0755' owner: root:root content: | #!/bin/bash -e - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - ATTEMPT_ONLINE_INSTALL=false - setup_complete=false - - OFFLINE_INSTALL_ATTEMPT_SLEEP=30 - MAX_OFFLINE_INSTALL_ATTEMPTS=40 - offline_attempts=1 - MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 - EJECT_ISO_FROM_OS={{ k8s.eject.iso }} - crucial_cmd_attempts=1 - iso_drive_path="" + MOUNT_ATTEMPT_SLEEP=15 + MOUNT_ATTEMPTS=1 while true; do - if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then - echo "Warning: Offline install timed out!" - break - fi set +e output=`blkid -o device -t TYPE=iso9660` set -e @@ -69,7 +132,6 @@ write_files: set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then - iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -80,159 +142,18 @@ write_files: if [ -d "$BINARIES_DIR" ]; then break fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" - sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP - offline_attempts=$[$offline_attempts + 1] + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $MOUNT_ATTEMPT_SLEEP seconds, attempt: $MOUNT_ATTEMPTS" + sleep $MOUNT_ATTEMPT_SLEEP + MOUNT_ATTEMPTS=$[$MOUNT_ATTEMPTS + 1] done + echo "Done mounting iso" - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - - if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + set +e + cp $ISO_MOUNT_DIR/scripts/* /opt/bin/ + echo "Done copying scripts" - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then - eject "${iso_drive_path}" - fi - fi - if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - ### Binaries not available offline ### - RELEASE="v1.16.3" - CNI_VERSION="v0.7.5" - CRICTL_VERSION="v1.16.0" - echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." - mkdir -p /opt/cni/bin - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz - - mkdir -p /opt/bin - curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - chmod +x {kubeadm,kubelet,kubectl} - - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - fi - - systemctl enable kubelet && systemctl start kubelet - modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - - if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Warning: kubeadm pull images failed after multiple tries!" - break; - fi - retval=0 - set +e - kubeadm config images pull --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - fi - - - path: /opt/bin/deploy-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - - if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then - echo "setup-kube-system is running!" - exit 1 - fi - modprobe ip_vs - modprobe ip_vs_wrr - modprobe ip_vs_sh - modprobe nf_conntrack - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification - - sudo touch /home/cloud/success - echo "true" > /home/cloud/success - - - path: /opt/bin/setup-containerd - permissions: '0755' - owner: root:root - content: | - #!/bin/bash -e - - export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" - export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" - - echo "creating config file for containerd" - containerd config default > /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml - sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - - echo "Restarting containerd service" - systemctl restart containerd - - - path: /etc/systemd/system/setup-kube-system.service + # Service files + - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' owner: root:root content: | @@ -240,32 +161,20 @@ write_files: Requires=containerd.service After=containerd.service - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - [Service] Type=simple StartLimitInterval=0 Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version ExecStart=/opt/bin/deploy-kube-system runcmd: - chown -R cloud:cloud /home/cloud/.ssh + - /opt/bin/mount_cks_iso + - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done + - until [ -f /opt/bin/deploy-kube-system ]; do sleep 5; done - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml + - /opt/bin/setup-kube-system - systemctl daemon-reload - systemctl restart containerd - - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] + - systemctl start deploy-kube-system diff --git a/scripts/cks/autoscale-kube-cluster b/scripts/cks/autoscale-kube-cluster new file mode 100755 index 000000000000..ac3fd8eab4ce --- /dev/null +++ b/scripts/cks/autoscale-kube-cluster @@ -0,0 +1,93 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function usage() { + cat << USAGE +Usage: ./autoscale-kube-cluster [OPTIONS]... +Enables autoscaling for the kubernetes cluster. +Arguments: + -i, --id string ID of the cluster + -e, --enable Enables autoscaling + -d, --disable Disables autoscaling + -M, --maxsize number Maximum size of the cluster + -m, --minsize number Minimum size of the cluster +Other arguments: + -h, --help Display this help message and exit +Examples: + ./autoscale-kube-cluster -e -M 3 -m 1 + ./autoscale-kube-cluster -d +USAGE + exit 0 +} +ID="" +ENABLE="" +MINSIZE="" +MAXSIZE="" +while [ -n "$1" ]; do + case "$1" in + -h | --help) + usage + ;; + -i | --id) + ID=$2 + shift 2 + ;; + -e | --enable) + ENABLE="true" + shift 1 + ;; + -d | --enable) + ENABLE="false" + shift 1 + ;; + -M | --maxsize) + MAXSIZE=$2 + shift 2 + ;; + -m | --minsize) + MINSIZE=$2 + shift 2 + ;; + -*|*) + echo "ERROR: no such option $1. -h or --help for help" + exit 1 + ;; + esac +done +if [ $ENABLE == "true" ] ; then + if [ -e /opt/autoscaler/autoscaler_tmpl.yaml ]; then + sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml + /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml + exit 0 + else + mkdir -p /opt/autoscaler + AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" + autoscaler_conf_file="/opt/autoscaler/autoscaler_tmpl.yaml" + curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} + if [ $? -ne 0 ]; then + echo "Unable to connect to the internet to download the autoscaler deployment and image" + exit 1 + else + sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml + /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml + exit 0 + fi + fi +else + /opt/bin/kubectl delete deployment -n kube-system cluster-autoscaler +fi diff --git a/scripts/cks/deploy-cloudstack-secret b/scripts/cks/deploy-cloudstack-secret new file mode 100755 index 000000000000..9356f8a03f14 --- /dev/null +++ b/scripts/cks/deploy-cloudstack-secret @@ -0,0 +1,68 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function usage() { + cat << USAGE +Usage: ./deploy-cloudstack-secret [OPTIONS]... +To deploy the keys needed for the cloudstack kubernetes provider. +Arguments: + -u, --url string ID of the cluster + -k, --key string API Key + -s, --secret string Secret Key +Other arguments: + -h, --help Display this help message and exit +Examples: + ./deploy-cloudstack-secret -u http://localhost:8080 -k abcd -s efgh +USAGE + exit 0 +} +API_URL="" +API_KEY="" +SECRET_KEY="" +while [ -n "$1" ]; do + case "$1" in + -h | --help) + usage + ;; + -u | --url) + API_URL=$2 + shift 2 + ;; + -k | --key) + API_KEY=$2 + shift 2 + ;; + -s | --secret) + SECRET_KEY=$2 + shift 2 + ;; + -*|*) + echo "ERROR: no such option $1. -h or --help for help" + exit 1 + ;; + esac +done +cat > /tmp/cloud-config < "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm init failed!" + exit 1 + fi + retval=0 + set +e + kubeadm init --token $CLUSTER_TOKEN --token-ttl 0 $INIT_ARGS --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + export KUBECONFIG=/etc/kubernetes/admin.conf + + mkdir -p /root/.kube + cp -i /etc/kubernetes/admin.conf /root/.kube/config + chown $(id -u):$(id -g) /root/.kube/config + echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then + ### Network, dashboard configs available offline ### + echo "Offline configs are available!" + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml + rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + else + /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" + /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml + fi + + /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true + /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true + /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true + ;; + + control-plane-add) + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + rm -rf /etc/kubernetes/pki + /usr/bin/curl -k https://$CLUSTER_JOIN_IP:6443/version + kubeadm join $CLUSTER_JOIN_IP:6443 --token $CLUSTER_TOKEN --control-plane --certificate-key $CLUSTER_HA_CERT_KEY --discovery-token-unsafe-skip-ca-verification + export KUBECONFIG=/etc/kubernetes/admin.conf + + set +e + mkdir -p /root/.kube + cp -i /etc/kubernetes/admin.conf /root/.kube/config + chown $(id -u):$(id -g) /root/.kube/config + echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + set -e + ;; + + worker) + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + rm -rf /etc/kubernetes/pki + /usr/bin/curl -k https://$CLUSTER_JOIN_IP:6443/version + kubeadm join $CLUSTER_JOIN_IP:6443 --token $CLUSTER_TOKEN --discovery-token-unsafe-skip-ca-verification + +esac + +sudo touch /home/cloud/success +echo "true" > /home/cloud/success diff --git a/scripts/cks/deploy-provider b/scripts/cks/deploy-provider new file mode 100755 index 000000000000..ce71e21072b6 --- /dev/null +++ b/scripts/cks/deploy-provider @@ -0,0 +1,36 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +(/opt/bin/kubectl get pods -A | grep cloud-controller-manager) && exit 0 + +if [ -e /opt/provider/provider.yaml ]; then + /opt/bin/kubectl apply -f /opt/provider/provider.yaml + exit 0 +else + mkdir -p /opt/provider + PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml" + provider_conf_file="/opt/provider/provider.yaml" + curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} + if [ $? -ne 0 ]; then + echo "Unable to connect to the internet to download the provider deployment and image" + exit 1 + else + /opt/bin/kubectl apply -f /opt/provider/provider.yaml + exit 0 + fi +fi diff --git a/scripts/cks/mount-cks-iso b/scripts/cks/mount-cks-iso new file mode 100755 index 000000000000..9c5bf6ebaa51 --- /dev/null +++ b/scripts/cks/mount-cks-iso @@ -0,0 +1,41 @@ +#!/bin/bash -e + +ISO_MOUNT_DIR=/mnt/k8sdisk +BINARIES_DIR=${ISO_MOUNT_DIR}/ +MOUNT_ATTEMPT_SLEEP=15 +MOUNT_ATTEMPTS=1 +while true; do + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $MOUNT_ATTEMPT_SLEEP seconds, attempt: $MOUNT_ATTEMPTS" + sleep $MOUNT_ATTEMPT_SLEEP + MOUNT_ATTEMPTS=$[$MOUNT_ATTEMPTS + 1] +done +echo "Done mounting iso" + +set +e +cp $ISO_MOUNT_DIR/scripts/* /opt/bin/ +echo "Done copying scripts" diff --git a/scripts/cks/setup-containerd-registry b/scripts/cks/setup-containerd-registry new file mode 100755 index 000000000000..3a19fc79172f --- /dev/null +++ b/scripts/cks/setup-containerd-registry @@ -0,0 +1,18 @@ +#!/bin/bash -e + +REGISTRY_URL=`cat /opt/cks/registry_url` +REGISTRY_URL_ENDPOINT=`cat /opt/cks/registry_url_endpoint` +REGISTRY_USERNAME=`cat /opt/cks/registry_username` +REGISTRY_PASSWORD=`cat /opt/cks/registry_password` +REGISTRY_TOKEN=`cat /opt/cks/registry_token` + +export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"$REGISTRY_URL_ENDPOINT\"]\n \\ endpoint = [\"$REGISTRY_URL\"]" +export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"$REGISTRY_URL_ENDPOINT\".auth]\n\tusername = \"$REGISTRY_USERNAME\" \n\tpassword = \"$REGISTRY_PASSWORD\" \n\tidentitytoken = \"$REGISTRY_TOKEN\"" + +echo "creating config file for containerd" +containerd config default > /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml +sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + +echo "Restarting containerd service" +systemctl restart containerd diff --git a/scripts/cks/setup-kube-system b/scripts/cks/setup-kube-system new file mode 100755 index 000000000000..28e224bbdcff --- /dev/null +++ b/scripts/cks/setup-kube-system @@ -0,0 +1,92 @@ +#!/bin/bash -e + +if [[ -f "/home/cloud/success" ]]; then +echo "Already provisioned!" +exit 0 +fi + +ISO_MOUNT_DIR=/mnt/k8sdisk +BINARIES_DIR=${ISO_MOUNT_DIR}/ +K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ +ATTEMPT_ONLINE_INSTALL=false +setup_complete=false + +OFFLINE_INSTALL_ATTEMPT_SLEEP=15 +MAX_OFFLINE_INSTALL_ATTEMPTS=100 +offline_attempts=1 +MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 +EJECT_ISO_FROM_OS=`cat /opt/cks/eject_iso` +crucial_cmd_attempts=1 +iso_drive_path=`blkid -o device -t TYPE=iso9660` + +if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin +fi + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + + mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi +fi + +PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` +echo $PAUSE_IMAGE +if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml +fi + +systemctl enable kubelet && systemctl start kubelet +modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 diff --git a/scripts/cks/upgrade-kubernetes.sh b/scripts/cks/upgrade-kubernetes.sh new file mode 100755 index 000000000000..b85ea00e9ed6 --- /dev/null +++ b/scripts/cks/upgrade-kubernetes.sh @@ -0,0 +1,143 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Version 1.14 and below needs extra flags with kubeadm upgrade node +if [ $# -lt 4 ]; then + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO" + echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false false" + exit 1 +fi +UPGRADE_VERSION="${1}" +IS_MAIN_CONTROL="" +if [ $# -gt 1 ]; then + IS_MAIN_CONTROL="${2}" +fi +IS_OLD_VERSION="" +if [ $# -gt 2 ]; then + IS_OLD_VERSION="${3}" +fi +EJECT_ISO_FROM_OS=false +if [ $# -gt 3 ]; then + EJECT_ISO_FROM_OS="${4}" +fi + +export PATH=$PATH:/opt/bin + +ISO_MOUNT_DIR=/mnt/k8sdisk +BINARIES_DIR=${ISO_MOUNT_DIR}/ + +OFFLINE_INSTALL_ATTEMPT_SLEEP=5 +MAX_OFFLINE_INSTALL_ATTEMPTS=10 +offline_attempts=1 +iso_drive_path="" +while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] +done + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + + cd /opt/bin + + cp ${BINARIES_DIR}/k8s/kubeadm /opt/bin + chmod +x kubeadm + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + ctr image import "${BINARIES_DIR}/docker/$line" + done <<< "$output" + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + + # Fetch the autoscaler if present + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then + set +e + kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} -y + retval=$? + set -e + if [ $retval -ne 0 ]; then + kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} --ignore-preflight-errors=CoreDNSUnsupportedPlugins -y + fi + else + if [ "${IS_OLD_VERSION}" == 'true' ]; then + kubeadm --v=5 upgrade node config --kubelet-version ${UPGRADE_VERSION} + else + kubeadm --v=5 upgrade node + fi + fi + + systemctl stop kubelet + cp -a ${BINARIES_DIR}/k8s/{kubelet,kubectl} /opt/bin + chmod +x {kubelet,kubectl} + systemctl restart kubelet + + if [ "${IS_MAIN_CONTROL}" == 'true' ]; then + /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml + /opt/bin/kubectl apply -f ${BINARIES_DIR}/dashboard.yaml + fi + + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi +fi diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index e7981d6ac0b0..f5af3d9be3a0 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -76,6 +76,17 @@ else curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} fi +echo "Downloading scripts" +script_files="autoscale-kube-cluster deploy-cloudstack-secret deploy-kube-system deploy-provider mount-cks-iso setup-containerd-registry setup-kube-system upgrade-kubernetes" +scripts_dir="${working_dir}/scripts" +mkdir -p ${scripts_dir} +for file in ${script_files} +do + curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack/main/scripts/cks/${file}" -o "${scripts_dir}/${file}" +done +chmod +x ${scripts_dir}/* + + NETWORK_CONFIG_URL="${5}" echo "Downloading network config ${NETWORK_CONFIG_URL}" network_conf_file="${working_dir}/network.yaml" From 363dc16a79eda26e97b96ad0117d5224579cdbca Mon Sep 17 00:00:00 2001 From: davidjumani Date: Mon, 30 May 2022 16:27:47 +0530 Subject: [PATCH 2/3] Revert "cks: Fix when deployed on a nw without internet access" This reverts commit 40e3338001b2f1c239533bedac5b60d8b9d2f631. --- ...esClusterResourceModifierActionWorker.java | 50 ++- .../KubernetesClusterStartWorker.java | 42 ++- .../resources/conf/k8s-control-node-add.yml | 272 +++++++++++++++ .../main/resources/conf/k8s-control-node.yml | 329 ++++++++++++++++++ .../src/main/resources/conf/k8s-node.yml | 289 +++++++++------ scripts/cks/autoscale-kube-cluster | 93 ----- scripts/cks/deploy-cloudstack-secret | 68 ---- scripts/cks/deploy-kube-system | 104 ------ scripts/cks/deploy-provider | 36 -- scripts/cks/mount-cks-iso | 41 --- scripts/cks/setup-containerd-registry | 18 - scripts/cks/setup-kube-system | 92 ----- scripts/cks/upgrade-kubernetes.sh | 143 -------- .../util/create-kubernetes-binaries-iso.sh | 11 - 14 files changed, 838 insertions(+), 750 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml delete mode 100755 scripts/cks/autoscale-kube-cluster delete mode 100755 scripts/cks/deploy-cloudstack-secret delete mode 100755 scripts/cks/deploy-kube-system delete mode 100755 scripts/cks/deploy-provider delete mode 100755 scripts/cks/mount-cks-iso delete mode 100755 scripts/cks/setup-containerd-registry delete mode 100755 scripts/cks/setup-kube-system delete mode 100755 scripts/cks/upgrade-kubernetes.sh diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 187c87330501..81db87a7c7de 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -135,22 +135,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected VolumeDao volumeDao; protected String kubernetesClusterNodeNamePrefix; - protected static final String apiServerCertPlaceholder = "{{ k8s_control_node.apiserver.crt }}"; - protected static final String apiServerKeyPlaceholder = "{{ k8s_control_node.apiserver.key }}"; - protected static final String caCertPlaceholder = "{{ k8s_control_node.ca.crt }}"; - protected static final String sshPubKeyPlaceholder = "{{ k8s.ssh.pub.key }}"; - protected static final String clusterTokenPlaceholder = "{{ k8s_control_node.cluster.token }}"; - protected static final String clusterInitArgsPlaceholder = "{{ k8s_control_node.cluster.initargs }}"; - protected static final String ejectIsoPlaceholder = "{{ k8s.eject.iso }}"; - protected static final String joinIpPlaceholder = "{{ k8s_control_node.join_ip }}"; - protected static final String nodeTypePlaceholder = "{{ k8s.node.type }}"; - protected static final String clusterHACertificateKeyPlaceholder = "{{ k8s_control_node.cluster.ha.certificate.key }}"; - protected static final String registryUrlPlaceholder = "{{ k8s.registry.url }}"; - protected static final String registryUrlEndpointPlaceholder = "{{ k8s.registry.url.endpoint }}"; - protected static final String registryUsernamePlaceholder = "{{ k8s.registry.username }}"; - protected static final String registryPasswordPlaceholder = "{{ k8s.registry.password }}"; - protected static final String registryTokenPlaceholder = "{{ k8s.registry.token }}"; - protected static final String cksUserdataFile = "/conf/k8s-node.yml"; protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -162,7 +146,11 @@ protected void init() { } private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sNodeConfig = readResourceFile(cksUserdataFile); + String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -171,11 +159,10 @@ private String getKubernetesNodeConfig(final String joinIp, final boolean ejectI pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sNodeConfig = k8sNodeConfig.replace(nodeTypePlaceholder, "worker"); - k8sNodeConfig = k8sNodeConfig.replace(sshPubKeyPlaceholder, pubKey); - k8sNodeConfig = k8sNodeConfig.replace(joinIpPlaceholder, joinIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sNodeConfig = k8sNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); + k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); + k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); @@ -204,18 +191,23 @@ protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details - String runCmd = "- bash -x /opt/bin/setup-containerd-registry"; + String runCmd = "- bash -x /opt/bin/setup-containerd"; - String registryEndpoint = registryUrl.split("://")[1]; + String registryEp = registryUrl.split("://")[1]; k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + final String registryUname = "{{registry.username}}"; + final String registryPsswd = "{{registry.password}}"; final String usernamePasswordKey = registryUsername + ":" + registryPassword; String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); - k8sConfig = k8sConfig.replace(registryUrlPlaceholder, registryUrl); - k8sConfig = k8sConfig.replace(registryUrlEndpointPlaceholder, registryEndpoint); - k8sConfig = k8sConfig.replace(registryUsernamePlaceholder, registryUsername); - k8sConfig = k8sConfig.replace(registryPasswordPlaceholder, registryPassword); - k8sConfig = k8sConfig.replace(registryTokenPlaceholder, base64Auth); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryUname, registryUsername); + k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); } return k8sConfig; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 2c8e9b9235bc..6612a7608325 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -133,7 +133,14 @@ private boolean isKubernetesVersionSupportsHA() { private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, final String hostName, final boolean haSupported, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile(cksUserdataFile); + String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); + final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; + final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; + final String caCert = "{{ k8s_control_node.ca.crt }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String clusterToken = "{{ k8s_control_node.cluster.token }}"; + final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); if (!serverIp.equals(controlNodeIp)) { @@ -145,9 +152,9 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); - k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCertPlaceholder, tlsClientCert.replace("\n", "\n ")); - k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKeyPlaceholder, tlsPrivateKey.replace("\n", "\n ")); - k8sControlNodeConfig = k8sControlNodeConfig.replace(caCertPlaceholder, tlsCaCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); + k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -156,9 +163,8 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlNodeConfig = k8sControlNodeConfig.replace(nodeTypePlaceholder, "control-plane"); - k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKeyPlaceholder, pubKey); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", @@ -168,8 +174,8 @@ private String getKubernetesControlNodeConfig(final String controlNodeIp, final } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsPlaceholder, initArgs); - k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; @@ -231,7 +237,12 @@ private UserVm createKubernetesControlNode(final Network network, String serverI } private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile(cksUserdataFile); + String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (StringUtils.isNotEmpty(sshKeyPair)) { @@ -240,12 +251,11 @@ private String getKubernetesAdditionalControlNodeConfig(final String joinIp, fin pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sControlNodeConfig = k8sControlNodeConfig.replace(nodeTypePlaceholder, "control-plane-add"); - k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKeyPlaceholder, pubKey); - k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpPlaceholder, joinIp); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenPlaceholder, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKeyPlaceholder, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); - k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoPlaceholder, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); + k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml new file mode 100644 index 000000000000..335ae9194e6d --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -0,0 +1,272 @@ +#cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +users: + - name: cloud + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + +write_files: + - path: /opt/bin/setup-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} + crucial_cmd_attempts=1 + iso_drive_path="" + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + fi + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi + + - path: /opt/bin/deploy-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + + - path: /etc/systemd/system/setup-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version + ExecStart=/opt/bin/deploy-kube-system + +runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd + - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done + - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml new file mode 100644 index 000000000000..76adc5a7d296 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -0,0 +1,329 @@ +#cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +users: + - name: cloud + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + +write_files: + - path: /etc/conf.d/nfs + permissions: '0644' + content: | + OPTS_RPC_MOUNTD="" + + - path: /etc/kubernetes/pki/cloudstack/ca.crt + permissions: '0644' + content: | + {{ k8s_control_node.ca.crt }} + + - path: /etc/kubernetes/pki/cloudstack/apiserver.crt + permissions: '0644' + content: | + {{ k8s_control_node.apiserver.crt }} + + - path: /etc/kubernetes/pki/cloudstack/apiserver.key + permissions: '0600' + content: | + {{ k8s_control_node.apiserver.key }} + + - path: /opt/bin/setup-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} + crucial_cmd_attempts=1 + iso_drive_path="" + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + fi + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi + + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm init failed!" + exit 1 + fi + retval=0 + set +e + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + + - path: /opt/bin/deploy-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + export KUBECONFIG=/etc/kubernetes/admin.conf + + mkdir -p /root/.kube + cp -i /etc/kubernetes/admin.conf /root/.kube/config + chown $(id -u):$(id -g) /root/.kube/config + echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + + if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then + ### Network, dashboard configs available offline ### + echo "Offline configs are available!" + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml + rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + else + /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" + /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml + fi + + /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true + /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true + /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true + + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + + - path: /etc/systemd/system/setup-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version + ExecStart=/opt/bin/deploy-kube-system + +runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - containerd config default > /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd + - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done + - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 6834d40874f6..86966245c83e 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -25,98 +25,35 @@ users: {{ k8s.ssh.pub.key }} write_files: - - # Control plane specific - - path: /etc/conf.d/nfs - permissions: '0644' - content: | - OPTS_RPC_MOUNTD="" - - - path: /etc/kubernetes/pki/cloudstack/apiserver.crt - permissions: '0644' - content: | - {{ k8s_control_node.apiserver.crt }} - - - path: /etc/kubernetes/pki/cloudstack/apiserver.key - permissions: '0600' - content: | - {{ k8s_control_node.apiserver.key }} - - - path: /etc/kubernetes/pki/cloudstack/ca.crt - permissions: '0644' - content: | - {{ k8s_control_node.ca.crt }} - - # Kubeadm extra args - - path: /opt/cks/cluster_token - permissions: '0644' - content: | - {{ k8s_control_node.cluster.token }} - - - path: /opt/cks/cluster_init_args - permissions: '0644' - content: | - {{ k8s_control_node.cluster.initargs }} - - - path: /opt/cks/cluster_join_ip - permissions: '0644' - content: | - {{ k8s_control_node.join_ip }} - - - path: /opt/cks/cluster_ha_cert_key - permissions: '0644' - content: | - {{ k8s_control_node.cluster.ha.certificate.key }} - - # Custom registry - - path: /opt/cks/registry_url - permissions: '0644' - content: | - {{ k8s.registry.url }} - - - path: /opt/cks/registry_url_endpoint - permissions: '0644' - content: | - {{ k8s.registry.url.endpoint }} - - - path: /opt/cks/registry_username - permissions: '0644' - content: | - {{ k8s.registry.username }} - - - path: /opt/cks/registry_password - permissions: '0644' - content: | - {{ k8s.registry.password }} - - - path: /opt/cks/registry_token - permissions: '0644' - content: | - {{ k8s.registry.token }} - - # Misc settings - - path: /opt/cks/node_type - permissions: '0644' - content: | - {{ k8s.node.type }} - - - path: /opt/cks/eject_iso - permissions: '0644' - content: | - {{ k8s.eject.iso }} - - # To get things started - - path: /opt/bin/mount_cks_iso - permissions: '0755' + - path: /opt/bin/setup-kube-system + permissions: '0700' owner: root:root content: | #!/bin/bash -e + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ - MOUNT_ATTEMPT_SLEEP=15 - MOUNT_ATTEMPTS=1 + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=30 + MAX_OFFLINE_INSTALL_ATTEMPTS=40 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} + crucial_cmd_attempts=1 + iso_drive_path="" while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi set +e output=`blkid -o device -t TYPE=iso9660` set -e @@ -132,6 +69,7 @@ write_files: set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -142,18 +80,159 @@ write_files: if [ -d "$BINARIES_DIR" ]; then break fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $MOUNT_ATTEMPT_SLEEP seconds, attempt: $MOUNT_ATTEMPTS" - sleep $MOUNT_ATTEMPT_SLEEP - MOUNT_ATTEMPTS=$[$MOUNT_ATTEMPTS + 1] + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] done - echo "Done mounting iso" - set +e - cp $ISO_MOUNT_DIR/scripts/* /opt/bin/ - echo "Done copying scripts" + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi - # Service files - - path: /etc/systemd/system/deploy-kube-system.service + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + ctr image import "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then + mkdir -p /opt/autoscaler + cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml + fi + if [ -e "${BINARIES_DIR}/provider.yaml" ]; then + mkdir -p /opt/provider + cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi + fi + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull --cri-socket /run/containerd/containerd.sock + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi + + - path: /opt/bin/deploy-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --discovery-token-unsafe-skip-ca-verification + + sudo touch /home/cloud/success + echo "true" > /home/cloud/success + + - path: /opt/bin/setup-containerd + permissions: '0755' + owner: root:root + content: | + #!/bin/bash -e + + export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\ endpoint = [\"{{registry.url}}\"]" + export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\"" + + echo "creating config file for containerd" + containerd config default > /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml + sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml + + echo "Restarting containerd service" + systemctl restart containerd + + - path: /etc/systemd/system/setup-kube-system.service permissions: '0755' owner: root:root content: | @@ -161,20 +240,32 @@ write_files: Requires=containerd.service After=containerd.service + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - path: /etc/systemd/system/deploy-kube-system.service + permissions: '0755' + owner: root:root + content: | + [Unit] + After=setup-kube-system.service + [Service] Type=simple StartLimitInterval=0 Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_control_node.join_ip }}:6443/version ExecStart=/opt/bin/deploy-kube-system runcmd: - chown -R cloud:cloud /home/cloud/.ssh - - /opt/bin/mount_cks_iso - - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done - - until [ -f /opt/bin/deploy-kube-system ]; do sleep 5; done - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - /opt/bin/setup-kube-system - systemctl daemon-reload - systemctl restart containerd - - systemctl start deploy-kube-system + - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done + - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done + - [ systemctl, start, setup-kube-system ] + - [ systemctl, start, deploy-kube-system ] diff --git a/scripts/cks/autoscale-kube-cluster b/scripts/cks/autoscale-kube-cluster deleted file mode 100755 index ac3fd8eab4ce..000000000000 --- a/scripts/cks/autoscale-kube-cluster +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -function usage() { - cat << USAGE -Usage: ./autoscale-kube-cluster [OPTIONS]... -Enables autoscaling for the kubernetes cluster. -Arguments: - -i, --id string ID of the cluster - -e, --enable Enables autoscaling - -d, --disable Disables autoscaling - -M, --maxsize number Maximum size of the cluster - -m, --minsize number Minimum size of the cluster -Other arguments: - -h, --help Display this help message and exit -Examples: - ./autoscale-kube-cluster -e -M 3 -m 1 - ./autoscale-kube-cluster -d -USAGE - exit 0 -} -ID="" -ENABLE="" -MINSIZE="" -MAXSIZE="" -while [ -n "$1" ]; do - case "$1" in - -h | --help) - usage - ;; - -i | --id) - ID=$2 - shift 2 - ;; - -e | --enable) - ENABLE="true" - shift 1 - ;; - -d | --enable) - ENABLE="false" - shift 1 - ;; - -M | --maxsize) - MAXSIZE=$2 - shift 2 - ;; - -m | --minsize) - MINSIZE=$2 - shift 2 - ;; - -*|*) - echo "ERROR: no such option $1. -h or --help for help" - exit 1 - ;; - esac -done -if [ $ENABLE == "true" ] ; then - if [ -e /opt/autoscaler/autoscaler_tmpl.yaml ]; then - sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml - /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml - exit 0 - else - mkdir -p /opt/autoscaler - AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" - autoscaler_conf_file="/opt/autoscaler/autoscaler_tmpl.yaml" - curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} - if [ $? -ne 0 ]; then - echo "Unable to connect to the internet to download the autoscaler deployment and image" - exit 1 - else - sed -e "s//$ID/g" -e "s//$MINSIZE/g" -e "s//$MAXSIZE/g" /opt/autoscaler/autoscaler_tmpl.yaml > /opt/autoscaler/autoscaler_now.yaml - /opt/bin/kubectl apply -f /opt/autoscaler/autoscaler_now.yaml - exit 0 - fi - fi -else - /opt/bin/kubectl delete deployment -n kube-system cluster-autoscaler -fi diff --git a/scripts/cks/deploy-cloudstack-secret b/scripts/cks/deploy-cloudstack-secret deleted file mode 100755 index 9356f8a03f14..000000000000 --- a/scripts/cks/deploy-cloudstack-secret +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -function usage() { - cat << USAGE -Usage: ./deploy-cloudstack-secret [OPTIONS]... -To deploy the keys needed for the cloudstack kubernetes provider. -Arguments: - -u, --url string ID of the cluster - -k, --key string API Key - -s, --secret string Secret Key -Other arguments: - -h, --help Display this help message and exit -Examples: - ./deploy-cloudstack-secret -u http://localhost:8080 -k abcd -s efgh -USAGE - exit 0 -} -API_URL="" -API_KEY="" -SECRET_KEY="" -while [ -n "$1" ]; do - case "$1" in - -h | --help) - usage - ;; - -u | --url) - API_URL=$2 - shift 2 - ;; - -k | --key) - API_KEY=$2 - shift 2 - ;; - -s | --secret) - SECRET_KEY=$2 - shift 2 - ;; - -*|*) - echo "ERROR: no such option $1. -h or --help for help" - exit 1 - ;; - esac -done -cat > /tmp/cloud-config < "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm init failed!" - exit 1 - fi - retval=0 - set +e - kubeadm init --token $CLUSTER_TOKEN --token-ttl 0 $INIT_ARGS --cri-socket /run/containerd/containerd.sock - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - export KUBECONFIG=/etc/kubernetes/admin.conf - - mkdir -p /root/.kube - cp -i /etc/kubernetes/admin.conf /root/.kube/config - chown $(id -u):$(id -g) /root/.kube/config - echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ - if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then - ### Network, dashboard configs available offline ### - echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml - rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - else - /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')" - /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml - fi - - /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true - /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true - /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true - ;; - - control-plane-add) - modprobe ip_vs - modprobe ip_vs_wrr - modprobe ip_vs_sh - modprobe nf_conntrack - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - rm -rf /etc/kubernetes/pki - /usr/bin/curl -k https://$CLUSTER_JOIN_IP:6443/version - kubeadm join $CLUSTER_JOIN_IP:6443 --token $CLUSTER_TOKEN --control-plane --certificate-key $CLUSTER_HA_CERT_KEY --discovery-token-unsafe-skip-ca-verification - export KUBECONFIG=/etc/kubernetes/admin.conf - - set +e - mkdir -p /root/.kube - cp -i /etc/kubernetes/admin.conf /root/.kube/config - chown $(id -u):$(id -g) /root/.kube/config - echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - set -e - ;; - - worker) - modprobe ip_vs - modprobe ip_vs_wrr - modprobe ip_vs_sh - modprobe nf_conntrack - if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin - fi - rm -rf /etc/kubernetes/pki - /usr/bin/curl -k https://$CLUSTER_JOIN_IP:6443/version - kubeadm join $CLUSTER_JOIN_IP:6443 --token $CLUSTER_TOKEN --discovery-token-unsafe-skip-ca-verification - -esac - -sudo touch /home/cloud/success -echo "true" > /home/cloud/success diff --git a/scripts/cks/deploy-provider b/scripts/cks/deploy-provider deleted file mode 100755 index ce71e21072b6..000000000000 --- a/scripts/cks/deploy-provider +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -(/opt/bin/kubectl get pods -A | grep cloud-controller-manager) && exit 0 - -if [ -e /opt/provider/provider.yaml ]; then - /opt/bin/kubectl apply -f /opt/provider/provider.yaml - exit 0 -else - mkdir -p /opt/provider - PROVIDER_URL="https://raw.githubusercontent.com/apache/cloudstack-kubernetes-provider/main/deployment.yaml" - provider_conf_file="/opt/provider/provider.yaml" - curl -sSL ${PROVIDER_URL} -o ${provider_conf_file} - if [ $? -ne 0 ]; then - echo "Unable to connect to the internet to download the provider deployment and image" - exit 1 - else - /opt/bin/kubectl apply -f /opt/provider/provider.yaml - exit 0 - fi -fi diff --git a/scripts/cks/mount-cks-iso b/scripts/cks/mount-cks-iso deleted file mode 100755 index 9c5bf6ebaa51..000000000000 --- a/scripts/cks/mount-cks-iso +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -e - -ISO_MOUNT_DIR=/mnt/k8sdisk -BINARIES_DIR=${ISO_MOUNT_DIR}/ -MOUNT_ATTEMPT_SLEEP=15 -MOUNT_ATTEMPTS=1 -while true; do - set +e - output=`blkid -o device -t TYPE=iso9660` - set -e - if [ "$output" != "" ]; then - while read -r line; do - if [ ! -d "${ISO_MOUNT_DIR}" ]; then - mkdir "${ISO_MOUNT_DIR}" - fi - retval=0 - set +e - mount -o ro "${line}" "${ISO_MOUNT_DIR}" - retval=$? - set -e - if [ $retval -eq 0 ]; then - if [ -d "$BINARIES_DIR" ]; then - break - else - umount "${line}" && rmdir "${ISO_MOUNT_DIR}" - fi - fi - done <<< "$output" - fi - if [ -d "$BINARIES_DIR" ]; then - break - fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $MOUNT_ATTEMPT_SLEEP seconds, attempt: $MOUNT_ATTEMPTS" - sleep $MOUNT_ATTEMPT_SLEEP - MOUNT_ATTEMPTS=$[$MOUNT_ATTEMPTS + 1] -done -echo "Done mounting iso" - -set +e -cp $ISO_MOUNT_DIR/scripts/* /opt/bin/ -echo "Done copying scripts" diff --git a/scripts/cks/setup-containerd-registry b/scripts/cks/setup-containerd-registry deleted file mode 100755 index 3a19fc79172f..000000000000 --- a/scripts/cks/setup-containerd-registry +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -REGISTRY_URL=`cat /opt/cks/registry_url` -REGISTRY_URL_ENDPOINT=`cat /opt/cks/registry_url_endpoint` -REGISTRY_USERNAME=`cat /opt/cks/registry_username` -REGISTRY_PASSWORD=`cat /opt/cks/registry_password` -REGISTRY_TOKEN=`cat /opt/cks/registry_token` - -export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"$REGISTRY_URL_ENDPOINT\"]\n \\ endpoint = [\"$REGISTRY_URL\"]" -export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"$REGISTRY_URL_ENDPOINT\".auth]\n\tusername = \"$REGISTRY_USERNAME\" \n\tpassword = \"$REGISTRY_PASSWORD\" \n\tidentitytoken = \"$REGISTRY_TOKEN\"" - -echo "creating config file for containerd" -containerd config default > /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml -sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml - -echo "Restarting containerd service" -systemctl restart containerd diff --git a/scripts/cks/setup-kube-system b/scripts/cks/setup-kube-system deleted file mode 100755 index 28e224bbdcff..000000000000 --- a/scripts/cks/setup-kube-system +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash -e - -if [[ -f "/home/cloud/success" ]]; then -echo "Already provisioned!" -exit 0 -fi - -ISO_MOUNT_DIR=/mnt/k8sdisk -BINARIES_DIR=${ISO_MOUNT_DIR}/ -K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ -ATTEMPT_ONLINE_INSTALL=false -setup_complete=false - -OFFLINE_INSTALL_ATTEMPT_SLEEP=15 -MAX_OFFLINE_INSTALL_ATTEMPTS=100 -offline_attempts=1 -MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 -EJECT_ISO_FROM_OS=`cat /opt/cks/eject_iso` -crucial_cmd_attempts=1 -iso_drive_path=`blkid -o device -t TYPE=iso9660` - -if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then - export PATH=$PATH:/opt/bin -fi - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - - mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - mkdir -p /opt/bin - cd /opt/bin - cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} . - chmod +x {kubeadm,kubelet,kubectl} - - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service - mkdir -p /etc/systemd/system/kubelet.service.d - sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf - - echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" - break; - fi - retval=0 - set +e - ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done - done <<< "$output" - setup_complete=true - fi - - mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then - eject "${iso_drive_path}" - fi -fi - -PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` -echo $PAUSE_IMAGE -if [ -n "$PAUSE_IMAGE" ]; then - sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml -fi - -systemctl enable kubelet && systemctl start kubelet -modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 diff --git a/scripts/cks/upgrade-kubernetes.sh b/scripts/cks/upgrade-kubernetes.sh deleted file mode 100755 index b85ea00e9ed6..000000000000 --- a/scripts/cks/upgrade-kubernetes.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Version 1.14 and below needs extra flags with kubeadm upgrade node -if [ $# -lt 4 ]; then - echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO" - echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false false" - exit 1 -fi -UPGRADE_VERSION="${1}" -IS_MAIN_CONTROL="" -if [ $# -gt 1 ]; then - IS_MAIN_CONTROL="${2}" -fi -IS_OLD_VERSION="" -if [ $# -gt 2 ]; then - IS_OLD_VERSION="${3}" -fi -EJECT_ISO_FROM_OS=false -if [ $# -gt 3 ]; then - EJECT_ISO_FROM_OS="${4}" -fi - -export PATH=$PATH:/opt/bin - -ISO_MOUNT_DIR=/mnt/k8sdisk -BINARIES_DIR=${ISO_MOUNT_DIR}/ - -OFFLINE_INSTALL_ATTEMPT_SLEEP=5 -MAX_OFFLINE_INSTALL_ATTEMPTS=10 -offline_attempts=1 -iso_drive_path="" -while true; do - if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then - echo "Warning: Offline install timed out!" - break - fi - set +e - output=`blkid -o device -t TYPE=iso9660` - set -e - if [ "$output" != "" ]; then - while read -r line; do - if [ ! -d "${ISO_MOUNT_DIR}" ]; then - mkdir "${ISO_MOUNT_DIR}" - fi - retval=0 - set +e - mount -o ro "${line}" "${ISO_MOUNT_DIR}" - retval=$? - set -e - if [ $retval -eq 0 ]; then - if [ -d "$BINARIES_DIR" ]; then - iso_drive_path="${line}" - break - else - umount "${line}" && rmdir "${ISO_MOUNT_DIR}" - fi - fi - done <<< "$output" - fi - if [ -d "$BINARIES_DIR" ]; then - break - fi - echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" - sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP - offline_attempts=$[$offline_attempts + 1] -done - -if [ -d "$BINARIES_DIR" ]; then - ### Binaries available offline ### - echo "Installing binaries from ${BINARIES_DIR}" - - cd /opt/bin - - cp ${BINARIES_DIR}/k8s/kubeadm /opt/bin - chmod +x kubeadm - - output=`ls ${BINARIES_DIR}/docker/` - if [ "$output" != "" ]; then - while read -r line; do - ctr image import "${BINARIES_DIR}/docker/$line" - done <<< "$output" - fi - if [ -e "${BINARIES_DIR}/provider.yaml" ]; then - mkdir -p /opt/provider - cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml - fi - - # Fetch the autoscaler if present - if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then - mkdir -p /opt/autoscaler - cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml - fi - - tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz - tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz - - if [ "${IS_MAIN_CONTROL}" == 'true' ]; then - set +e - kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} -y - retval=$? - set -e - if [ $retval -ne 0 ]; then - kubeadm --v=5 upgrade apply ${UPGRADE_VERSION} --ignore-preflight-errors=CoreDNSUnsupportedPlugins -y - fi - else - if [ "${IS_OLD_VERSION}" == 'true' ]; then - kubeadm --v=5 upgrade node config --kubelet-version ${UPGRADE_VERSION} - else - kubeadm --v=5 upgrade node - fi - fi - - systemctl stop kubelet - cp -a ${BINARIES_DIR}/k8s/{kubelet,kubectl} /opt/bin - chmod +x {kubelet,kubectl} - systemctl restart kubelet - - if [ "${IS_MAIN_CONTROL}" == 'true' ]; then - /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml - /opt/bin/kubectl apply -f ${BINARIES_DIR}/dashboard.yaml - fi - - umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then - eject "${iso_drive_path}" - fi -fi diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index f5af3d9be3a0..e7981d6ac0b0 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -76,17 +76,6 @@ else curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} fi -echo "Downloading scripts" -script_files="autoscale-kube-cluster deploy-cloudstack-secret deploy-kube-system deploy-provider mount-cks-iso setup-containerd-registry setup-kube-system upgrade-kubernetes" -scripts_dir="${working_dir}/scripts" -mkdir -p ${scripts_dir} -for file in ${script_files} -do - curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack/main/scripts/cks/${file}" -o "${scripts_dir}/${file}" -done -chmod +x ${scripts_dir}/* - - NETWORK_CONFIG_URL="${5}" echo "Downloading network config ${NETWORK_CONFIG_URL}" network_conf_file="${working_dir}/network.yaml" From c48bcce4f7d4fb09b40ca59bb6b0ee9474c3787b Mon Sep 17 00:00:00 2001 From: davidjumani Date: Mon, 30 May 2022 16:35:37 +0530 Subject: [PATCH 3/3] cks: Fix issue when creating cluster in nw without internet access --- .../resources/conf/k8s-control-node-add.yml | 35 +++++------ .../main/resources/conf/k8s-control-node.yml | 62 +++++++++---------- .../src/main/resources/conf/k8s-node.yml | 34 +++++----- .../resources/script/upgrade-kubernetes.sh | 11 +++- 4 files changed, 71 insertions(+), 71 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 335ae9194e6d..b4167147ba48 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -120,7 +120,7 @@ write_files: fi retval=0 set +e - ctr image import "${BINARIES_DIR}/docker/$line" + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -139,6 +139,15 @@ write_files: mkdir -p /opt/provider cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -230,9 +239,10 @@ write_files: sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml echo "Restarting containerd service" + systemctl daemon-reload systemctl restart containerd - - path: /etc/systemd/system/setup-kube-system.service + - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' owner: root:root content: | @@ -240,18 +250,6 @@ write_files: Requires=containerd.service After=containerd.service - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - [Service] Type=simple StartLimitInterval=0 @@ -263,10 +261,7 @@ runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - systemctl daemon-reload - - systemctl restart containerd + - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done + - /opt/bin/setup-kube-system - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] - + - systemctl start deploy-kube-system diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 76adc5a7d296..d4f0cd74ef39 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -140,7 +140,7 @@ write_files: fi retval=0 set +e - ctr image import "${BINARIES_DIR}/docker/$line" + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -161,6 +161,15 @@ write_files: mkdir -p /opt/provider cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -210,6 +219,22 @@ write_files: done fi + - path: /opt/bin/deploy-kube-system + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 while true; do if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then @@ -227,17 +252,6 @@ write_files: crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] done - - path: /opt/bin/deploy-kube-system - permissions: '0700' - owner: root:root - content: | - #!/bin/bash -e - - if [[ -f "/home/cloud/success" ]]; then - echo "Already provisioned!" - exit 0 - fi - K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then @@ -287,9 +301,10 @@ write_files: sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml echo "Restarting containerd service" + systemctl daemon-reload systemctl restart containerd - - path: /etc/systemd/system/setup-kube-system.service + - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' owner: root:root content: | @@ -297,33 +312,18 @@ write_files: Requires=containerd.service After=containerd.service - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - [Service] Type=simple StartLimitInterval=0 Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version ExecStart=/opt/bin/deploy-kube-system runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - systemctl daemon-reload - - systemctl restart containerd + - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done + - /opt/bin/setup-kube-system - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] + - systemctl start deploy-kube-system diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 86966245c83e..de1f4c9ffc70 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -120,7 +120,7 @@ write_files: fi retval=0 set +e - ctr image import "${BINARIES_DIR}/docker/$line" + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" retval=$? set -e if [ $retval -eq 0 ]; then @@ -139,6 +139,15 @@ write_files: mkdir -p /opt/provider cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml fi + + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + systemctl daemon-reload + systemctl restart containerd + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" @@ -230,9 +239,10 @@ write_files: sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml echo "Restarting containerd service" + systemctl daemon-reload systemctl restart containerd - - path: /etc/systemd/system/setup-kube-system.service + - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' owner: root:root content: | @@ -240,18 +250,6 @@ write_files: Requires=containerd.service After=containerd.service - [Service] - Type=simple - StartLimitInterval=0 - ExecStart=/opt/bin/setup-kube-system - - - path: /etc/systemd/system/deploy-kube-system.service - permissions: '0755' - owner: root:root - content: | - [Unit] - After=setup-kube-system.service - [Service] Type=simple StartLimitInterval=0 @@ -263,9 +261,7 @@ runcmd: - chown -R cloud:cloud /home/cloud/.ssh - containerd config default > /etc/containerd/config.toml - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml - - systemctl daemon-reload - - systemctl restart containerd + - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done + - /opt/bin/setup-kube-system - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done - - until [ -f /etc/systemd/system/setup-kube-system.service ]; do sleep 5; done - - [ systemctl, start, setup-kube-system ] - - [ systemctl, start, deploy-kube-system ] + - systemctl start deploy-kube-system diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index b85ea00e9ed6..80ea10df1f01 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -93,7 +93,7 @@ if [ -d "$BINARIES_DIR" ]; then output=`ls ${BINARIES_DIR}/docker/` if [ "$output" != "" ]; then while read -r line; do - ctr image import "${BINARIES_DIR}/docker/$line" + ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line" done <<< "$output" fi if [ -e "${BINARIES_DIR}/provider.yaml" ]; then @@ -107,6 +107,12 @@ if [ -d "$BINARIES_DIR" ]; then cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml fi + PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1` + echo $PAUSE_IMAGE + if [ -n "$PAUSE_IMAGE" ]; then + sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml + fi + tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz @@ -129,6 +135,9 @@ if [ -d "$BINARIES_DIR" ]; then systemctl stop kubelet cp -a ${BINARIES_DIR}/k8s/{kubelet,kubectl} /opt/bin chmod +x {kubelet,kubectl} + + systemctl daemon-reload + systemctl restart containerd systemctl restart kubelet if [ "${IS_MAIN_CONTROL}" == 'true' ]; then