Skip to content

Commit 0a374aa

Browse files
JSponJSpon
committed
Allow for scripts in CKS ISO to fully control CKS deployment
1 parent 21af134 commit 0a374aa

25 files changed

Lines changed: 1411 additions & 0 deletions

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,22 @@ write_files:
8989
export PATH=$PATH:/opt/bin
9090
fi
9191
92+
if [ -f "${BINARIES_DIR}/scripts/control-node-add/setup-kube-system" ]; then
93+
# This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system for the second stage
94+
if /bin/bash ${BINARIES_DIR}/scripts/control-node-add/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.ha.certificate.key }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
95+
rc=0
96+
echo "Success running setup-kube-system on a control plane node add from ISO."
97+
else
98+
rc=$?
99+
echo "Failed running setup-kube-system on a control plane node add from ISO."
100+
fi
101+
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
102+
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
103+
eject "${iso_drive_path}"
104+
fi
105+
exit $rc
106+
fi
107+
92108
if [ -d "$BINARIES_DIR" ]; then
93109
### Binaries available offline ###
94110
echo "Installing binaries from ${BINARIES_DIR}"

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,22 @@ write_files:
109109
export PATH=$PATH:/opt/bin
110110
fi
111111
112+
if [ -f "${BINARIES_DIR}/scripts/control-node/setup-kube-system" ]; then
113+
# This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc
114+
if /bin/bash ${BINARIES_DIR}/scripts/control-node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.cluster.token }}" "{{ k8s_control_node.cluster.initargs }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
115+
rc=0
116+
echo "Success running setup-kube-system on control plane node from ISO."
117+
else
118+
rc=$?
119+
echo "Failed running setup-kube-system on control plane node from ISO."
120+
fi
121+
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
122+
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
123+
eject "${iso_drive_path}"
124+
fi
125+
exit $rc
126+
fi
127+
112128
if [ -d "$BINARIES_DIR" ]; then
113129
### Binaries available offline ###
114130
echo "Installing binaries from ${BINARIES_DIR}"

‎plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,22 @@ write_files:
8989
export PATH=$PATH:/opt/bin
9090
fi
9191
92+
if [ -f "${BINARIES_DIR}/scripts/node/setup-kube-system" ]; then
93+
# This setup script will copy any necessary files out of the iso, and possibly overwrite /opt/bin/deploy-kube-system, /opt/bin/setup-containerd, etc
94+
if /bin/bash ${BINARIES_DIR}/scripts/node/setup-kube-system ${BINARIES_DIR} "{{ k8s_control_node.join_ip }}" "{{ k8s_control_node.cluster.token }}" "{{registry.url}}" "{{registry.url.endpoint}}" "{{registry.username}}" "{{registry.password}}" "{{registry.token}}"; then
95+
rc=0
96+
echo "Success running setup-kube-system on worker node from ISO."
97+
else
98+
rc=$?
99+
echo "Failed running setup-kube-system on worker node from ISO."
100+
fi
101+
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
102+
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
103+
eject "${iso_drive_path}"
104+
fi
105+
exit $rc
106+
fi
107+
92108
if [ -d "$BINARIES_DIR" ]; then
93109
### Binaries available offline ###
94110
echo "Installing binaries from ${BINARIES_DIR}"

‎scripts/util/cks-samples/README.md‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# CloudStack Kubernetes Service Cloud-init Overrides - Examples
2+
3+
Example files that can be used when creating the CloudStack Kubernetes Service ISO file to override the normal behavior of the Kubernetes cluster creation
4+
5+
6+
## Usage
7+
8+
### The basic directory
9+
10+
In the basic directory, there is a scripts directory that can be used as a template to start the customization. It is basically the original cloud-init scripts, but they can then be modified as needed for the CKS cluster. To start, just copy the scripts directory into the util directory, peer with the create-kubernetes-binaries-iso.sh script.
11+
12+
### The cilium directory
13+
14+
In the cilium directory, there is a modified version of create-kubernetes-binaries-iso.sh which contains what would be needed to install Helm and install Cilium instead of Weave. Helm needs to be installed and in the path on the OS where the ISO build will be run. There is no need to customize anything under the cilium directory if you are only looking to run Cilium instead of Weave, but just like the basic directory, other aspects can be customized.
15+
16+
17+
## References in CloudStack
18+
19+
The following cloud-init scripts will call the setup-kube-system scripts found here when the cluster is created:
20+
21+
* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
22+
* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
23+
* plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
24+

‎scripts/util/cks-samples/basic/scripts/control-node-add/deploy-kube-system.tmpl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash -e
2+
3+
if [[ -f "/home/cloud/success" ]]; then
4+
echo "Already provisioned!"
5+
exit 0
6+
fi
7+
8+
if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
9+
echo "setup-kube-system is running!"
10+
exit 1
11+
fi
12+
modprobe ip_vs
13+
modprobe ip_vs_wrr
14+
modprobe ip_vs_sh
15+
modprobe nf_conntrack
16+
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
17+
export PATH=$PATH:/opt/bin
18+
fi
19+
kubeadm join {{ k8s_control_node.join_ip }}:6443 --token {{ k8s_control_node.cluster.token }} --control-plane --certificate-key {{ k8s_control_node.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification
20+
21+
mkdir -p /root/.kube
22+
cp -i /etc/kubernetes/admin.conf /root/.kube/config
23+
chown $(id -u):$(id -g) /root/.kube/config
24+
25+
sudo touch /home/cloud/success
26+
echo "true" > /home/cloud/success
27+

‎scripts/util/cks-samples/basic/scripts/control-node-add/setup-containerd.tmpl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash -e
2+
3+
export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]"
4+
export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""
5+
6+
echo "creating config file for containerd"
7+
containerd config default > /etc/containerd/config.toml
8+
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
9+
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml
10+
11+
echo "Restarting containerd service"
12+
systemctl daemon-reload
13+
systemctl restart containerd

‎scripts/util/cks-samples/basic/scripts/control-node-add/setup-kube-system‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
#!/bin/bash -e
2+
3+
if [ $# -lt 9 ]; then
4+
echo "Usage: /bin/bash $0 BINARIES_DIR CLUSTER_JOIN_IP CLUSTER_TOKEN CERTIFICATE_KEY REGISTRY_URL REGISTRY_URL_ENDPOINT REGISTRY_USERNAME REGISTRY_PASSWORD REGISTRY_TOKEN" >&2
5+
exit 1
6+
fi
7+
8+
BINARIES_DIR="$1"
9+
CLUSTER_JOIN_IP="$2"
10+
CLUSTER_TOKEN="$3"
11+
CERTIFICATE_KEY="$4"
12+
REGISTRY_URL="$5"
13+
REGISTRY_URL_ENDPOINT="$6"
14+
REGISTRY_USERNAME="$7"
15+
REGISTRY_PASSWORD="$8"
16+
REGISTRY_TOKEN="$9"
17+
18+
ATTEMPT_ONLINE_INSTALL=false
19+
setup_complete=false
20+
21+
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
22+
crucial_cmd_attempts=1
23+
24+
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
25+
export PATH=$PATH:/opt/bin
26+
fi
27+
28+
if [ -f ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl ]; then
29+
sed -e "s/{{ k8s_control_node.cluster.token }}/${CLUSTER_TOKEN}/g" -e "s/{{ k8s_control_node.cluster.ha.certificate.key }}/${CERTIFICATE_KEY}/g" -e "s/{{ k8s_control_node.join_ip }}/${CLUSTER_JOIN_IP}/g" ${BINARIES_DIR}/scripts/control-node-add/deploy-kube-system.tmpl > /opt/bin/deploy-kube-system
30+
fi
31+
32+
if [ -f ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl ]; then
33+
sed -e "s/{{registry.url}}/${REGISTRY_URL}/g" -e "s/{{registry.url.endpoint}}/${REGISTRY_URL_ENDPOINT}/g" -e "s/{{registry.username}}/${REGISTRY_USERNAME}/g" -e "s/{{registry.password}}/${REGISTRY_PASSWORD}/g" -e "s/{{registry.token}}/${REGISTRY_TOKEN}/g" ${BINARIES_DIR}/scripts/control-node-add/setup-containerd.tmpl > /opt/bin/setup-containerd
34+
fi
35+
36+
if [ -d "$BINARIES_DIR" ]; then
37+
### Binaries available offline ###
38+
echo "Installing binaries from ${BINARIES_DIR}"
39+
mkdir -p /opt/cni/bin
40+
tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz
41+
42+
mkdir -p /opt/bin
43+
tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz
44+
45+
mkdir -p /opt/bin
46+
cd /opt/bin
47+
cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} .
48+
chmod +x {kubeadm,kubelet,kubectl}
49+
50+
sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service
51+
mkdir -p /etc/systemd/system/kubelet.service.d
52+
sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
53+
54+
echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet
55+
56+
output=`ls ${BINARIES_DIR}/docker/`
57+
if [ "$output" != "" ]; then
58+
while read -r line; do
59+
crucial_cmd_attempts=1
60+
while true; do
61+
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
62+
echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!"
63+
break;
64+
fi
65+
retval=0
66+
set +e
67+
ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line"
68+
retval=$?
69+
set -e
70+
if [ $retval -eq 0 ]; then
71+
break;
72+
fi
73+
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
74+
done
75+
done <<< "$output"
76+
setup_complete=true
77+
fi
78+
if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then
79+
mkdir -p /opt/autoscaler
80+
cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml
81+
fi
82+
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
83+
mkdir -p /opt/provider
84+
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
85+
fi
86+
87+
PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1`
88+
echo $PAUSE_IMAGE
89+
if [ -n "$PAUSE_IMAGE" ]; then
90+
sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
91+
fi
92+
systemctl daemon-reload
93+
systemctl restart containerd
94+
95+
fi
96+
if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
97+
### Binaries not available offline ###
98+
RELEASE="v1.16.3"
99+
CNI_VERSION="v0.7.5"
100+
CRICTL_VERSION="v1.16.0"
101+
echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet."
102+
mkdir -p /opt/cni/bin
103+
curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz
104+
105+
mkdir -p /opt/bin
106+
curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz
107+
108+
mkdir -p /opt/bin
109+
cd /opt/bin
110+
curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
111+
chmod +x {kubeadm,kubelet,kubectl}
112+
113+
curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service
114+
mkdir -p /etc/systemd/system/kubelet.service.d
115+
curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
116+
fi
117+
118+
systemctl enable kubelet && systemctl start kubelet
119+
modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1
120+
121+
if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
122+
crucial_cmd_attempts=1
123+
while true; do
124+
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
125+
echo "Warning: kubeadm pull images failed after multiple tries!"
126+
break;
127+
fi
128+
retval=0
129+
set +e
130+
kubeadm config images pull --cri-socket /run/containerd/containerd.sock
131+
retval=$?
132+
set -e
133+
if [ $retval -eq 0 ]; then
134+
break;
135+
fi
136+
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
137+
done
138+
fi

‎scripts/util/cks-samples/basic/scripts/control-node/deploy-kube-system.tmpl‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash -e
2+
3+
if [[ -f "/home/cloud/success" ]]; then
4+
echo "Already provisioned!"
5+
exit 0
6+
fi
7+
8+
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
9+
export PATH=$PATH:/opt/bin
10+
fi
11+
12+
MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
13+
crucial_cmd_attempts=1
14+
while true; do
15+
if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
16+
echo "Error: kubeadm init failed!"
17+
exit 1
18+
fi
19+
retval=0
20+
set +e
21+
kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
22+
retval=$?
23+
set -e
24+
if [ $retval -eq 0 ]; then
25+
break;
26+
fi
27+
crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
28+
done
29+
30+
K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/
31+
32+
if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
33+
echo "setup-kube-system is running!"
34+
exit 1
35+
fi
36+
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
37+
export PATH=$PATH:/opt/bin
38+
fi
39+
export KUBECONFIG=/etc/kubernetes/admin.conf
40+
41+
mkdir -p /root/.kube
42+
cp -i /etc/kubernetes/admin.conf /root/.kube/config
43+
chown $(id -u):$(id -g) /root/.kube/config
44+
echo export PATH=\$PATH:/opt/bin >> /root/.bashrc
45+
46+
if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
47+
### Network, dashboard configs available offline ###
48+
echo "Offline configs are available!"
49+
/opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml
50+
/opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml
51+
rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
52+
else
53+
/opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')"
54+
/opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml
55+
fi
56+
57+
/opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true
58+
/opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true
59+
/opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true
60+
61+
sudo touch /home/cloud/success
62+
echo "true" > /home/cloud/success

‎scripts/util/cks-samples/basic/scripts/control-node/setup-containerd.tmpl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash -e
2+
3+
export registryConfig="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"*\"]\n \\ endpoint = [\"{{registry.url}}\"]"
4+
export registryCredentials="\\ [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""
5+
6+
echo "creating config file for containerd"
7+
containerd config default > /etc/containerd/config.toml
8+
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
9+
sed -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml
10+
11+
echo "Restarting containerd service"
12+
systemctl daemon-reload
13+
systemctl restart containerd

0 commit comments

Comments
 (0)