Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions .ci/chart_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ FUNCTION=${FUNCTION:-"false"}
MANAGER=${MANAGER:-"false"}
ALLOW_LOADBALANCERS=${ALLOW_LOADBALANCERS:-"false"}

source ${PULSAR_HOME}/.ci/helm.sh
# shellcheck source=.ci/helm.sh
source "${PULSAR_HOME}"/.ci/helm.sh

# create cluster
ci::create_cluster
Expand All @@ -46,11 +47,11 @@ while [[ $# -gt 0 ]]; do
shift
done

if [[ "x${SYMMETRIC}" == "xtrue" ]]; then
if [[ "${SYMMETRIC}" == "true" ]]; then
extra_opts+=("-s")
fi

if [[ "x${EXTRA_SUPERUSERS}" != "x" ]]; then
if [[ "${EXTRA_SUPERUSERS}" != "" ]]; then
extra_opts+=("--pulsar-superusers" "proxy-admin,broker-admin,admin,${EXTRA_SUPERUSERS}")
fi

Expand All @@ -62,9 +63,9 @@ if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then
PULSAR_CHART_VERSION="$UPGRADE_FROM_VERSION"

# Install Prometheus Operator CRDs using the upgrade script since kube-prometheus-stack is now disabled before the upgrade
${PULSAR_HOME}/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh
"${PULSAR_HOME}"/scripts/kube-prometheus-stack/upgrade_prometheus_operator_crds.sh

ci::install_pulsar_chart install ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} --set kube-prometheus-stack.enabled=false "${extra_opts[@]}"
ci::install_pulsar_chart install "${PULSAR_HOME}"/.ci/values-common.yaml "${PULSAR_HOME}"/"${VALUES_FILE}" --set kube-prometheus-stack.enabled=false "${extra_opts[@]}"
install_type="upgrade"
echo "Wait 10 seconds"
sleep 10
Expand All @@ -78,13 +79,13 @@ if [[ "$UPGRADE_FROM_VERSION" != "" ]]; then

if [[ "$(ci::helm_values_for_deployment | yq .victoria-metrics-k8s-stack.enabled)" == "true" ]]; then
echo "Upgrade Victoria Metrics Operator CRDs before upgrading the deployment"
${PULSAR_HOME}/scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
"${PULSAR_HOME}"/scripts/victoria-metrics-k8s-stack/upgrade_vm_operator_crds.sh
fi
fi

PULSAR_CHART_VERSION="local"
# install (or upgrade) pulsar chart
ci::install_pulsar_chart ${install_type} ${PULSAR_HOME}/.ci/values-common.yaml ${PULSAR_HOME}/${VALUES_FILE} "${extra_opts[@]}"
ci::install_pulsar_chart "${install_type}" "${PULSAR_HOME}"/.ci/values-common.yaml "${PULSAR_HOME}"/"${VALUES_FILE}" "${extra_opts[@]}"

echo "Wait 10 seconds"
sleep 10
Expand Down
6 changes: 3 additions & 3 deletions .ci/configure_ci_runner_for_debugging.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ if [[ -z "${GITHUB_ACTIONS}" ]]; then
exit 1
fi

cat >> $HOME/.bashrc <<'EOF'
cat >> "$HOME"/.bashrc <<'EOF'
function use_kind_kubeconfig() {
export KUBECONFIG=$(ls $HOME/kind/pulsar-ci-*/kubeconfig.yaml)
}
Expand Down Expand Up @@ -34,8 +34,8 @@ function k9s() {

alias k=kubectl
EOF
cat >> $HOME/.bash_profile <<'EOF'
cat >> "$HOME"/.bash_profile <<'EOF'
if [ -f ~/.bashrc ]; then
source ~/.bashrc
fi
EOF
EOF
256 changes: 134 additions & 122 deletions .ci/helm.sh

Large diffs are not rendered by default.

40 changes: 21 additions & 19 deletions hack/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,16 @@ function hack::ensure_kubectl() {
fi
echo "Installing kubectl v$KUBECTL_VERSION..."
tmpfile=$(mktemp)
trap "test -f $tmpfile && rm $tmpfile" RETURN
curl --retry 10 -L -o $tmpfile https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl
mv $tmpfile $KUBECTL_BIN
chmod +x $KUBECTL_BIN
trap 'test -f "$tmpfile" && rm "$tmpfile"' RETURN
curl --retry 10 -L -o "$tmpfile" https://dl.k8s.io/release/v"${KUBECTL_VERSION}"/bin/"${OS}"/"${ARCH}"/kubectl
mv "$tmpfile" "$KUBECTL_BIN"
chmod +x "$KUBECTL_BIN"
}

function hack::verify_helm() {
if test -x "$HELM_BIN"; then
local v=$($HELM_BIN version --short --client | grep -o -E '[0-9]+\.[0-9]+\.[0-9]+')
local v
v=$($HELM_BIN version --short --client | grep -o -E '[0-9]+\.[0-9]+\.[0-9]+')
[[ "$v" == "$HELM_VERSION" ]]
return
fi
Expand All @@ -88,8 +89,9 @@ function hack::ensure_helm() {
if hack::verify_helm; then
return 0
fi
local HELM_URL=https://get.helm.sh/helm-v${HELM_VERSION}-${OS}-${ARCH}.tar.gz
curl --retry 10 -L -s "$HELM_URL" | tar --strip-components 1 -C $OUTPUT_BIN -zxf - ${OS}-${ARCH}/helm
local HELM_URL
HELM_URL=https://get.helm.sh/helm-v${HELM_VERSION}-${OS}-${ARCH}.tar.gz
curl --retry 10 -L -s "$HELM_URL" | tar --strip-components 1 -C "$OUTPUT_BIN" -zxf - "${OS}"-"${ARCH}"/helm
}

function hack::verify_kind() {
Expand All @@ -106,10 +108,10 @@ function hack::ensure_kind() {
fi
echo "Installing kind v$KIND_VERSION..."
tmpfile=$(mktemp)
trap "test -f $tmpfile && rm $tmpfile" RETURN
curl --retry 10 -L -o $tmpfile https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-${OS}-${ARCH}
mv $tmpfile $KIND_BIN
chmod +x $KIND_BIN
trap 'test -f "$tmpfile" && rm "$tmpfile"' RETURN
curl --retry 10 -L -o "$tmpfile" https://github.com/kubernetes-sigs/kind/releases/download/v"${KIND_VERSION}"/kind-"${OS}"-"${ARCH}"
mv "$tmpfile" "$KIND_BIN"
chmod +x "$KIND_BIN"
}

# hack::version_ge "$v1" "$v2" checks whether "v1" is greater or equal to "v2"
Expand All @@ -131,16 +133,16 @@ function hack::ensure_cr() {
fi
echo "Installing chart-releaser ${CR_VERSION} ..."
tmpfile=$(mktemp)
trap "test -f $tmpfile && rm $tmpfile" RETURN
echo curl --retry 10 -L -o $tmpfile https://github.com/helm/chart-releaser/releases/download/v${CR_VERSION}/chart-releaser_${CR_VERSION}_${OS}_${ARCH}.tar.gz
curl --retry 10 -L -o $tmpfile https://github.com/helm/chart-releaser/releases/download/v${CR_VERSION}/chart-releaser_${CR_VERSION}_${OS}_${ARCH}.tar.gz
mv $tmpfile $CR_BIN
chmod +x $CR_BIN
trap 'test -f "$tmpfile" && rm "$tmpfile"' RETURN
echo curl --retry 10 -L -o "$tmpfile" https://github.com/helm/chart-releaser/releases/download/v"${CR_VERSION}"/chart-releaser_"${CR_VERSION}"_"${OS}"_"${ARCH}".tar.gz
curl --retry 10 -L -o "$tmpfile" https://github.com/helm/chart-releaser/releases/download/v"${CR_VERSION}"/chart-releaser_"${CR_VERSION}"_"${OS}"_"${ARCH}".tar.gz
mv "$tmpfile" "$CR_BIN"
chmod +x "$CR_BIN"
$CR_BIN version
}

function hack::ensure_kubeconform() {
echo "Installing kubeconform v$KUBECONFORM_VERSION..."
curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v${KUBECONFORM_VERSION}/kubeconform-${OS}-${ARCH}.tar.gz | tar -xzO kubeconform > $KUBECONFORM_BIN
chmod +x $KUBECONFORM_BIN
}
curl -s --retry 10 -L https://github.com/yannh/kubeconform/releases/download/v"${KUBECONFORM_VERSION}"/kubeconform-"${OS}"-"${ARCH}".tar.gz | tar -xzO kubeconform > "$KUBECONFORM_BIN"
chmod +x "$KUBECONFORM_BIN"
}
43 changes: 22 additions & 21 deletions hack/kind-cluster-build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@
# under the License.
#

PULSAR_CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd ${PULSAR_CHART_HOME}
PULSAR_CHART_HOME=$(unset CDPATH && cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
cd "${PULSAR_CHART_HOME}" || exit

source ${PULSAR_CHART_HOME}/hack/common.sh
# shellcheck source=hack/common.sh
source "${PULSAR_CHART_HOME}/hack/common.sh"

hack::ensure_kubectl
hack::ensure_helm
Expand Down Expand Up @@ -107,18 +108,18 @@ done

echo "############# start create cluster:[${clusterName}] #############"
workDir=${HOME}/kind/${clusterName}
mkdir -p ${workDir}
mkdir -p "${workDir}"

data_dir=${workDir}/data

echo "clean data dir: ${data_dir}"
if [ -d ${data_dir} ]; then
rm -rf ${data_dir}
if [ -d "${data_dir}" ]; then
rm -rf "${data_dir}"
fi

configFile=${workDir}/kind-config.yaml

cat <<EOF > ${configFile}
cat <<EOF > "${configFile}"
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
networking:
Expand All @@ -132,39 +133,39 @@ nodes:
protocol: TCP
EOF

for ((i=0;i<${nodeNum};i++))
for ((i=0;i<nodeNum;i++))
do
mkdir -p ${data_dir}/worker${i}
cat <<EOF >> ${configFile}
mkdir -p "${data_dir}"/worker${i}
cat <<EOF >> "${configFile}"
- role: worker
extraMounts:
EOF
for ((k=1;k<=${volumeNum};k++))
for ((k=1;k<=volumeNum;k++))
do
mkdir -p ${data_dir}/worker${i}/vol${k}
cat <<EOF >> ${configFile}
mkdir -p "${data_dir}"/worker${i}/vol${k}
cat <<EOF >> "${configFile}"
- containerPath: /mnt/disks/vol${k}
hostPath: ${data_dir}/worker${i}/vol${k}
EOF
done
done

matchedCluster=$(kind get clusters | grep ${clusterName})
matchedCluster=$(kind get clusters | grep "${clusterName}")
if [[ "${matchedCluster}" == "${clusterName}" ]]; then
echo "Kind cluster ${clusterName} already exists"
kind delete cluster --name=${clusterName}
kind delete cluster --name="${clusterName}"
fi
echo "start to create k8s cluster"
kind create cluster --config ${configFile} --image kindest/node:${k8sVersion} --name=${clusterName} --verbosity 3
kind create cluster --config "${configFile}" --image kindest/node:"${k8sVersion}" --name="${clusterName}" --verbosity 3
export KUBECONFIG=${workDir}/kubeconfig.yaml
kind get kubeconfig --name=${clusterName} > ${KUBECONFIG}
kind get kubeconfig --name="${clusterName}" > "${KUBECONFIG}"

echo "deploy docker registry in kind"
registryNode=${clusterName}-control-plane
registryNodeIP=$($KUBECTL_BIN get nodes ${registryNode} -o template --template='{{range.status.addresses}}{{if eq .type "InternalIP"}}{{.address}}{{end}}{{end}}')
registryNodeIP=$($KUBECTL_BIN get nodes "${registryNode}" -o template --template='{{range.status.addresses}}{{if eq .type "InternalIP"}}{{.address}}{{end}}{{end}}')
registryFile=${workDir}/registry.yaml

cat <<EOF >${registryFile}
cat <<EOF > "${registryFile}"
apiVersion: apps/v1
kind: DaemonSet
metadata:
Expand Down Expand Up @@ -232,14 +233,14 @@ spec:
- tcp-listen:5000,fork,reuseaddr
- tcp-connect:${registryNodeIP}:5000
EOF
$KUBECTL_BIN apply -f ${registryFile}
$KUBECTL_BIN apply -f "${registryFile}"

echo "############# success create cluster:[${clusterName}] #############"

echo "To start using your cluster, run:"
echo " export KUBECONFIG=${KUBECONFIG}"
echo ""
echo <<EOF
cat <<EOF
NOTE: In kind, nodes run docker network and cannot access host network.
If you configured local HTTP proxy in your docker, images may cannot be pulled
because http proxy is inaccessible.
Expand Down
3 changes: 1 addition & 2 deletions scripts/cert-manager/install-cert-manager.sh
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
Expand All @@ -17,8 +18,6 @@
# under the License.
#

#!/usr/bin/env bash

# If installation fails, the build should fail
set -e

Expand Down
8 changes: 4 additions & 4 deletions scripts/pulsar/cleanup_helm_release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
# under the License.
#

CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
cd ${CHART_HOME}
CHART_HOME=$(unset CDPATH && cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
cd "${CHART_HOME}" || exit

usage() {
cat <<EOF
Expand Down Expand Up @@ -73,12 +73,12 @@ release=${release:-pulsar-dev}

function delete_namespace() {
if [[ "${delete_namespace}" == "true" ]]; then
kubectl delete namespace ${namespace}
kubectl delete namespace "${namespace}"
fi
}

# delete tokens
kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs kubectl delete secrets -n ${namespace}
kubectl get secrets -n "${namespace}" | grep "${release}-token-" | awk '{print $1}' | xargs kubectl delete secrets -n "${namespace}"

# delete namespace
delete_namespace
4 changes: 2 additions & 2 deletions scripts/pulsar/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@ function validate_gke_required_tools(){
command -v "${comm}" > /dev/null 2>&1 || "need_${comm}"
done

gcloud container clusters list --project $PROJECT >/dev/null 2>&1 || { echo >&2 "Gcloud seems to be configured incorrectly or authentication is unsuccessfull"; exit 1; }
gcloud container clusters list --project "$PROJECT" >/dev/null 2>&1 || { echo >&2 "Gcloud seems to be configured incorrectly or authentication is unsuccessfull"; exit 1; }

}

function cluster_admin_password_gke(){
gcloud container clusters describe $CLUSTER_NAME --zone $ZONE --project $PROJECT --format='value(masterAuth.password)';
gcloud container clusters describe "$CLUSTER_NAME" --zone "$ZONE" --project "$PROJECT" --format='value(masterAuth.password)';
}

function validate_eks_required_tools(){
Expand Down
3 changes: 2 additions & 1 deletion scripts/pulsar/common_auth.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,5 @@ if [ -z "$PULSAR_VERSION" ]; then
PULSAR_VERSION="4.0.3"
fi
fi
PULSAR_TOKENS_CONTAINER_IMAGE="apachepulsar/pulsar:${PULSAR_VERSION}"
# shellcheck disable=SC2034
PULSAR_TOKENS_CONTAINER_IMAGE="apachepulsar/pulsar:${PULSAR_VERSION}"
Loading
Loading