From 400220b643394d25fccbc432b765840f81c41b4a Mon Sep 17 00:00:00 2001 From: Ed Cook Date: Mon, 27 Jul 2020 11:43:50 +0100 Subject: [PATCH 1/8] Add kapp-local-deploy, including local build of images Also switch to separate image values in ytt data to better enable bumping of image, see github.com/cloudfoundry/uaa for an example of this pattern --- .gitignore | 5 ++--- Makefile | 5 ++++- build/kbld.yaml | 13 +++++++++++++ build/rbac-proxy-relocation.yaml | 3 +++ deployments/k8s/manifests/deployment.yaml | 4 ++-- deployments/k8s/manifests/webhook.yaml | 2 +- .../k8s/{values.yaml => values/_default.yaml} | 4 ++++ deployments/k8s/values/images.yaml | 5 +++++ scripts/kapp-delete | 9 +++++++++ scripts/kapp-deploy | 18 ++++++++++++++++-- 10 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 build/kbld.yaml create mode 100644 build/rbac-proxy-relocation.yaml rename deployments/k8s/{values.yaml => values/_default.yaml} (90%) create mode 100644 deployments/k8s/values/images.yaml create mode 100755 scripts/kapp-delete diff --git a/.gitignore b/.gitignore index 7fad29e3..0e647ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - # Binaries for programs and plugins *.exe *.exe~ @@ -9,12 +8,10 @@ bin # Test binary, build with `go test -c` *.test - # Output of the go coverage tool, specifically when used with LiteIDE *.out # Kubernetes Generated files - skip generated files, except for vendored files - !vendor/**/zz_generated.* # editor and IDE paraphernalia @@ -22,4 +19,6 @@ bin *.swp *.swo *~ + .envrc +build/*.lock.copied diff --git a/Makefile b/Makefile index b9cfee9d..b333bac4 100644 --- a/Makefile +++ b/Makefile @@ -84,5 +84,8 @@ format: kapp-deploy: ./scripts/kapp-deploy +kapp-local-deploy: + ./scripts/kapp-deploy --local + kapp-delete: - kapp delete -a projects-operator -y + ./scripts/kapp-delete diff --git a/build/kbld.yaml b/build/kbld.yaml new file mode 100644 index 00000000..fbafedfc --- /dev/null +++ b/build/kbld.yaml @@ -0,0 +1,13 @@ +#@ load("@ytt:data", "data") +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: Sources +sources: +- imageRepo: dev.registry.pivotal.io/developer-console/projects-operator + path: . +--- +apiVersion: kbld.k14s.io/v1alpha1 +kind: ImageDestinations +destinations: +- imageRepo: dev.registry.pivotal.io/developer-console/projects-operator + newImage: #@ data.values.registry.hostname + '/' + data.values.registry.project + '/dev-projects-operator' diff --git a/build/rbac-proxy-relocation.yaml b/build/rbac-proxy-relocation.yaml new file mode 100644 index 00000000..f6d205e6 --- /dev/null +++ b/build/rbac-proxy-relocation.yaml @@ -0,0 +1,3 @@ +--- +images: +- image: dev.registry.pivotal.io/developer-console/kube-rbac-proxy:v0.4.1 diff --git a/deployments/k8s/manifests/deployment.yaml b/deployments/k8s/manifests/deployment.yaml index 3d5f6d33..0d304da5 100644 --- a/deployments/k8s/manifests/deployment.yaml +++ b/deployments/k8s/manifests/deployment.yaml @@ -28,7 +28,7 @@ spec: - --upstream=http://127.0.0.1:8080/ - --logtostderr=true - --v=10 - image: #@ data.values.registry.hostname + '/' + data.values.registry.project + "/kube-rbac-proxy:v0.4.1" + image: #@ data.values.images.kubeRbacProxy name: kube-rbac-proxy ports: - containerPort: 8443 @@ -43,7 +43,7 @@ spec: value: #@ data.values.clusterRoleRef - name: MAX_CONCURRENT_RECONCILES value: #@ data.values.maxConcurrentReconciles - image: #@ data.values.registry.hostname + '/' + data.values.registry.project + "/projects-operator:" + data.values.version + image: #@ data.values.images.projectsOperator name: manager resources: #@ data.values.resources nodeSelector: #@ data.values.nodeSelector diff --git a/deployments/k8s/manifests/webhook.yaml b/deployments/k8s/manifests/webhook.yaml index 33403cbf..9a8f134a 100644 --- a/deployments/k8s/manifests/webhook.yaml +++ b/deployments/k8s/manifests/webhook.yaml @@ -52,7 +52,7 @@ spec: - name: #@ data.values.registry.secretName containers: - name: webhook - image: #@ data.values.registry.hostname + '/' + data.values.registry.project + "/projects-operator:" + data.values.version + image: #@ data.values.images.projectsOperator command: - /webhook env: diff --git a/deployments/k8s/values.yaml b/deployments/k8s/values/_default.yaml similarity index 90% rename from deployments/k8s/values.yaml rename to deployments/k8s/values/_default.yaml index fd1ae72e..df42f216 100644 --- a/deployments/k8s/values.yaml +++ b/deployments/k8s/values/_default.yaml @@ -12,6 +12,10 @@ registry: password: secretName: "registry-secret" +images: + projectsOperator: + kubeRbacProxy: + clusterRoleRef: maxConcurrentReconciles: "4" diff --git a/deployments/k8s/values/images.yaml b/deployments/k8s/values/images.yaml new file mode 100644 index 00000000..3f9d230c --- /dev/null +++ b/deployments/k8s/values/images.yaml @@ -0,0 +1,5 @@ +#@data/values +--- +images: + projectsOperator: dev.registry.pivotal.io/developer-console/projects-operator@sha256:3ce47c158ece4a1d63d449e6152bfb848db7ae207c94ae2ef89748a46e1688c3 + kubeRbacProxy: dev.registry.pivotal.io/developer-console/kube-rbac-proxy:v0.4.1 diff --git a/scripts/kapp-delete b/scripts/kapp-delete new file mode 100755 index 00000000..413b1d28 --- /dev/null +++ b/scripts/kapp-delete @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +set -euo pipefail + +NAMESPACE="${NAMESPACE:-"projects-operator"}" + +kapp delete -y -a projects-operator -n "$NAMESPACE" diff --git a/scripts/kapp-deploy b/scripts/kapp-deploy index 7418f745..9d0e13a3 100755 --- a/scripts/kapp-deploy +++ b/scripts/kapp-deploy @@ -26,7 +26,21 @@ metadata: name: $NAMESPACE EOF -ytt -f $DIR/../deployments/k8s \ +LOCAL_YTT_FLAGS="" +LOCAL_KBLD_FLAGS="" +if [[ $# -eq 1 && "$1" =~ "--local" ]]; then + # use kbld to relocate the kube-rbac-proxy image to custom registry + kbld -f build/rbac-proxy-relocation.yaml | + kbld relocate \ + --repository "$REGISTRY_HOSTNAME/$REGISTRY_PROJECT/kube-rbac-proxy" \ + --lock-output build/rbac-proxy-relocation.lock.copied -f - + LOCAL_KBLD_FLAGS="-f ${DIR}/../build/rbac-proxy-relocation.lock.copied" + + # use kbld to build, tag and push the cf-service-bridge image to the custom registry. + LOCAL_YTT_FLAGS="-f ${DIR}/../build/kbld.yaml" +fi + +ytt -f $DIR/../deployments/k8s $LOCAL_YTT_FLAGS \ -v name=$NAME \ -v instance=$INSTANCE \ -v namespace=$NAMESPACE \ @@ -40,5 +54,5 @@ ytt -f $DIR/../deployments/k8s \ --data-value-file tls.cert=/tmp/webhook-server-tls.crt \ --data-value-file tls.key=/tmp/webhook-server-tls.key \ --data-value-file caCert=/tmp/ca.pem | \ -kbld -f - | \ +kbld $LOCAL_KBLD_FLAGS -f - | \ kapp deploy -y -a projects-operator -n $NAMESPACE -f - From 7577be2ca6a349198b8fdaee478abe6c2389c249 Mon Sep 17 00:00:00 2001 From: Ed Cook Date: Mon, 27 Jul 2020 11:55:48 +0100 Subject: [PATCH 2/8] Add automatic bumping of image sha to ci --- ci/pipeline.yml | 15 +++++++++++++++ ci/tasks/bump-image-ref.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 ci/tasks/bump-image-ref.yml diff --git a/ci/pipeline.yml b/ci/pipeline.yml index d8eac173..2d453035 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -96,6 +96,8 @@ jobs: passed: [build] params: format: oci + - task: bump-image-ref + file: projects-operator/ci/tasks/bump-image-ref.yml - task: kapp-deploy file: projects-operator/ci/tasks/kapp-deploy.yml params: @@ -114,6 +116,9 @@ jobs: CLUSTER_API_LOCATION: {{CLUSTER_API_LOCATION}} CLUSTER_NAME: ci-projects-operator DEVELOPER_PASSWORD: {{DEVELOPER_PASSWORD}} + - put: projects-operator-bump + params: + repository: projects-operator - name: promote-final plan: @@ -164,6 +169,16 @@ jobs: resources: - name: projects-operator + type: git + icon: git + source: + private_key: {{PROJECTS_OPERATOR_DEPLOY_KEY}} + uri: git@github.com:pivotal/projects-operator + branch: master + ignore_paths: + - deployments/k8s/values/images.yaml + +- name: projects-operator-bump type: git icon: git source: diff --git a/ci/tasks/bump-image-ref.yml b/ci/tasks/bump-image-ref.yml new file mode 100644 index 00000000..5beb120b --- /dev/null +++ b/ci/tasks/bump-image-ref.yml @@ -0,0 +1,29 @@ +platform: linux + +image_resource: + type: registry-image + source: + repository: ismteam/ci + tag: latest + +inputs: +- name: projects-operator +- name: projects-operator-image + +outputs: +- name: projects-operator + +run: + path: /bin/bash + args: + - -euc + - | + DIGEST=$(cat projects-operator-image/digest) + + pushd projects-operator + sed -i "s/projects-operator@.*/projects-operator@${DIGEST}/" deployments/k8s/values/images.yaml + git config user.email "pvtl-marketplace-demand+ci@vmware.com" + git config user.name "projects-operator-ci" + git add . + git commit -m "Bump projects-operator image digest" + popd From f56dc51a46c79edc3d670bea90fabb83502ecaed Mon Sep 17 00:00:00 2001 From: Ed Cook Date: Mon, 27 Jul 2020 11:56:06 +0100 Subject: [PATCH 3/8] Ensure that CI env var is available in generate-certs --- scripts/kapp-deploy | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/kapp-deploy b/scripts/kapp-deploy index 9d0e13a3..157e5acc 100755 --- a/scripts/kapp-deploy +++ b/scripts/kapp-deploy @@ -17,6 +17,7 @@ REGISTRY_SECRET_NAME="${REGISTRY_SECRET_NAME:-registry-secret}" CLUSTER_ROLE_REF="${CLUSTER_ROLE_REF:?"please set the cluster role ref"}" +export CI $DIR/generate-certs $NAME $INSTANCE $NAMESPACE cat < Date: Mon, 27 Jul 2020 12:11:24 +0100 Subject: [PATCH 4/8] Update install instructions and simplify default install Default install simplified by removing need to set REGISTRY_HOSTNAME and REGISTRY_PROJECT when using make kapp-deploy --- README.md | 42 +++++++++++++++++++++++++----------------- scripts/kapp-deploy | 4 ++-- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 56df2f92..940623d7 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,16 @@ controller. `Projects` are intended to provide isolation of kubernetes resources on a single kubernetes cluster. A `Project` is essentially a kubernetes namespace along with a corresponding set of RBAC rules. -## Installation and Usage +## Installation -`projects-operator` is currently deployed using [k14s](https://k14s.io). +### Prerequisites -You must first create a `ClusterRole` that contains the RBAC +* **kubectl**: For installation instructions, see [Install and Set Up kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) in the Kubernetes documentation. +* **Docker CLI**: For installation instructions, see the [Docker documentation](https://docs.docker.com/install). +* **k14s tools**: For installation instructions, see [k14s.io](https://k14s.io). +* **A Kubernetes cluster**: This will be where the CF Service Bridge components get installed. + +You must also create a `ClusterRole` that contains the RBAC rules you wish to be applied to each created `Project`. For example: ```yaml @@ -27,33 +32,36 @@ rules: verbs: - "*" ``` +The env var `CLUSTE_ROLE_REF` should be set to the name of the ClusterRole. -### Install - -Then you will need to build and push the projects-operator image to a registry. +### Standard install -```bash -$ docker build -t //projects-operator . -$ docker push //projects-operator +The default image registry for cf-service-bridge is dev.registry.pivotal.io. If +you have access to this registry, then you need only set the `INSTANCE`, `REGISTRY_USERNAME`, +`REGISTRY_PASSWORD` and `CLUSTER_ROLE_REF` env vars, then run `./scripts/kapp-deploy`. -# For example, docker build -t gcr.io/team-a/projects-operator . -``` +### Custom install -Then finally you can run the [/scripts/kapp-deploy](/scripts/kapp-deploy) script -to deploy projects-operator. +If you don't have access to the default registry, or if you are working on +cf-service-bridge and wish to deploy your local changes (i.e. for testing), we +will need to build and push the image to a custom registry. This can be done by +setting the following env vars: ```bash - export INSTANCE= export REGISTRY_HOSTNAME= # e.g. "gcr.io", "my.private.harbor.com", etc. export REGISTRY_PROJECT= # e.g. "team-a", "dev", etc. export REGISTRY_USERNAME= export REGISTRY_PASSWORD= export CLUSTER_ROLE_REF=my-clusterrole-with-rbac-for-each-project - -$ ./scripts/kapp-deploy ``` +Then run `make kapp-local-deploy`. NB: you will need to +have a docker daemon running and you will need to have run `docker login` for +the registry you are using. + +## Using Projects Operator + ### Creating a Project Apply projects yaml with a project name and a list of users/groups/serviceaccounts who have access, for example: @@ -77,7 +85,7 @@ spec: ### Uninstall ```bash -kapp -n delete -a projects-operator +make kapp-delete ``` ### Webhooks diff --git a/scripts/kapp-deploy b/scripts/kapp-deploy index 157e5acc..bdb2eb78 100755 --- a/scripts/kapp-deploy +++ b/scripts/kapp-deploy @@ -9,8 +9,8 @@ INSTANCE="${INSTANCE:?"please set the instance name of this deployment"}" NAMESPACE="${NAMESPACE:-"projects-operator"}" VERSION="${VERSION:-"latest"}" -REGISTRY_HOSTNAME="${REGISTRY_HOSTNAME:?"please set the hostname of your registry"}" -REGISTRY_PROJECT="${REGISTRY_PROJECT:?"please set the project of your registry"}" +REGISTRY_HOSTNAME="${REGISTRY_HOSTNAME:-"dev.registry.pivotal.io"}" +REGISTRY_PROJECT="${REGISTRY_PROJECT:-"developer-console"}" REGISTRY_USERNAME="${REGISTRY_USERNAME:?"please set the username of your registry"}" REGISTRY_PASSWORD="${REGISTRY_PASSWORD:?"please set the password of your registry"}" REGISTRY_SECRET_NAME="${REGISTRY_SECRET_NAME:-registry-secret}" From 36b831e0480dede10a1e7eb72bc1e45ce7f4d029 Mon Sep 17 00:00:00 2001 From: Ed Cook Date: Mon, 27 Jul 2020 15:14:18 +0100 Subject: [PATCH 5/8] Clean up kapp-deploy and kapp-delete in line with shellcheck --- scripts/kapp-delete | 2 -- scripts/kapp-deploy | 24 ++++++++++++------------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/kapp-delete b/scripts/kapp-delete index 413b1d28..dc5f6745 100755 --- a/scripts/kapp-delete +++ b/scripts/kapp-delete @@ -1,7 +1,5 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - set -euo pipefail NAMESPACE="${NAMESPACE:-"projects-operator"}" diff --git a/scripts/kapp-deploy b/scripts/kapp-deploy index bdb2eb78..3e31de56 100755 --- a/scripts/kapp-deploy +++ b/scripts/kapp-deploy @@ -18,7 +18,7 @@ REGISTRY_SECRET_NAME="${REGISTRY_SECRET_NAME:-registry-secret}" CLUSTER_ROLE_REF="${CLUSTER_ROLE_REF:?"please set the cluster role ref"}" export CI -$DIR/generate-certs $NAME $INSTANCE $NAMESPACE +$DIR/generate-certs "$NAME" "$INSTANCE" "$NAMESPACE" cat < Date: Wed, 29 Jul 2020 11:11:33 +0100 Subject: [PATCH 6/8] Switch kube-rbac-proxy image to be pinned by sha --- deployments/k8s/values/images.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/k8s/values/images.yaml b/deployments/k8s/values/images.yaml index 3f9d230c..1129c7ee 100644 --- a/deployments/k8s/values/images.yaml +++ b/deployments/k8s/values/images.yaml @@ -2,4 +2,4 @@ --- images: projectsOperator: dev.registry.pivotal.io/developer-console/projects-operator@sha256:3ce47c158ece4a1d63d449e6152bfb848db7ae207c94ae2ef89748a46e1688c3 - kubeRbacProxy: dev.registry.pivotal.io/developer-console/kube-rbac-proxy:v0.4.1 + kubeRbacProxy: dev.registry.pivotal.io/developer-console/kube-rbac-proxy@sha256:fc13323a791c633939b2cb9277b2914cf998dc2575ce16b497fa62871a65b81a From 08053697f9bb56bdb1f4393519af73dbb82309cf Mon Sep 17 00:00:00 2001 From: Ed Cook Date: Mon, 3 Aug 2020 14:48:34 +0100 Subject: [PATCH 7/8] When changing version, use the new values file [#173681350] Signed-off-by: Glen Rodgers --- ci/scripts/assemble-build-artefact.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/assemble-build-artefact.sh b/ci/scripts/assemble-build-artefact.sh index 4cb7501c..00be9a52 100755 --- a/ci/scripts/assemble-build-artefact.sh +++ b/ci/scripts/assemble-build-artefact.sh @@ -2,7 +2,7 @@ VERSION=$(cat version/version) -sed -i "s/version:.*/version:\ ${VERSION}/" projects-operator/deployments/k8s/values.yaml +sed -i "s/version:.*/version:\ ${VERSION}/" projects-operator/deployments/k8s/values/_default.yaml tar -czf projects-operator-${VERSION}.tgz -C projects-operator/deployments --transform "s/k8s/projects-operator/g" k8s/ mv projects-operator-${VERSION}.tgz archive/projects-operator-${VERSION}.tgz \ No newline at end of file From 0960faf6e108666bc1953f5195abbe4489b8ecfa Mon Sep 17 00:00:00 2001 From: Glen Rodgers Date: Mon, 3 Aug 2020 14:52:17 +0100 Subject: [PATCH 8/8] Improve bashing in scripts Signed-off-by: Ed Cook --- ci/scripts/assemble-build-artefact.sh | 8 ++++---- ci/scripts/kapp-deploy.sh | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/scripts/assemble-build-artefact.sh b/ci/scripts/assemble-build-artefact.sh index 00be9a52..ee738d82 100755 --- a/ci/scripts/assemble-build-artefact.sh +++ b/ci/scripts/assemble-build-artefact.sh @@ -1,8 +1,8 @@ -#!/bin/sh +#!/usr/bin/env bash VERSION=$(cat version/version) -sed -i "s/version:.*/version:\ ${VERSION}/" projects-operator/deployments/k8s/values/_default.yaml +sed -i "s/version:.*/version:\ ${VERSION}/" "projects-operator/deployments/k8s/values/_default.yaml" -tar -czf projects-operator-${VERSION}.tgz -C projects-operator/deployments --transform "s/k8s/projects-operator/g" k8s/ -mv projects-operator-${VERSION}.tgz archive/projects-operator-${VERSION}.tgz \ No newline at end of file +tar -czf "projects-operator-${VERSION}.tgz" -C "projects-operator/deployments" --transform "s/k8s/projects-operator/g" "k8s/" +mv "projects-operator-${VERSION}.tgz" "archive/projects-operator-${VERSION}.tgz" \ No newline at end of file diff --git a/ci/scripts/kapp-deploy.sh b/ci/scripts/kapp-deploy.sh index dd031828..d31057c3 100755 --- a/ci/scripts/kapp-deploy.sh +++ b/ci/scripts/kapp-deploy.sh @@ -1,15 +1,15 @@ -#!/bin/sh +#!/usr/bin/env bash -export PATH="$PATH:/root/go/bin" +export PATH="${PATH}:/root/go/bin" if [ -n "$KUBECONFIG_FILE_CONTENTS" ]; then - mkdir -p "$HOME/.kube" - echo "$KUBECONFIG_FILE_CONTENTS" > "$HOME/.kube/config" + mkdir -p "${HOME}/.kube" + echo "$KUBECONFIG_FILE_CONTENTS" > "${HOME}/.kube/config" fi -mkdir -p $HOME/.docker -DOCKER_AUTH=$(echo -n "$REGISTRY_USERNAME:$REGISTRY_PASSWORD" | base64 - | tr -d '\n') -cat < $HOME/.docker/config.json +mkdir -p "${HOME}/.docker" +DOCKER_AUTH=$(echo -n "${REGISTRY_USERNAME}:${REGISTRY_PASSWORD}" | base64 - | tr -d '\n') +cat < "${HOME}/.docker/config.json" { "auths": { "${REGISTRY_HOSTNAME}/${REGISTRY_PROJECT}": {