diff --git a/.env.template b/.env.template new file mode 100644 index 000000000..ef7a87f93 --- /dev/null +++ b/.env.template @@ -0,0 +1,3 @@ +# PORCH_GHCR_PREFIX_URL=/kptdev/krm-functions-catalog +# DOCKERHUB_MIRROR= +# DB_MAVEN_MIRROR= diff --git a/Makefile b/Makefile index c2ffa36a8..a2ac0527f 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,13 @@ ifneq ("$(wildcard .env)", "") export endif +export PORCH_GHCR_PREFIX_URL ?= ghcr.io/kptdev/krm-functions-catalog +# remove '/' suffix +export PORCH_GHCR_PREFIX_URL := $(patsubst %/,%,$(PORCH_GHCR_PREFIX_URL)) +export DOCKERHUB_MIRROR ?= +# remove '/' suffix +export DOCKERHUB_MIRROR := $(patsubst %/,%,$(DOCKERHUB_MIRROR)) + # Include module makefiles include make/build.mk # generate, tidy, porch, porchctl, build-images, push-images include make/deploy.mk # deploy, run-in-kind*, destroy, deployment-config*, load-images-to-kind, reload-* diff --git a/build/Dockerfile b/build/Dockerfile index f67a1c338..f4124e2d2 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - ARG ALPINE_VERSION=latest ARG GOLANG_BOOKWORM_VERSION=latest +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:${GOLANG_BOOKWORM_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_BOOKWORM_VERSION} AS builder WORKDIR /go/src @@ -43,7 +43,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ go build -trimpath -ldflags="-s -w" -v -o /porch ./cmd/porch -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/build/Dockerfile.apiserver b/build/Dockerfile.apiserver index 4bd54e121..7df36a211 100644 --- a/build/Dockerfile.apiserver +++ b/build/Dockerfile.apiserver @@ -14,8 +14,9 @@ ARG ALPINE_VERSION=latest ARG GOLANG_BOOKWORM_VERSION=latest +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:${GOLANG_BOOKWORM_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_BOOKWORM_VERSION} AS builder WORKDIR /workspace/src RUN git clone https://github.com/kubernetes/kubernetes --branch v1.30.3 --depth=1 @@ -23,7 +24,7 @@ WORKDIR /workspace/src/kubernetes RUN apt-get update && apt-get install --yes rsync RUN make kube-apiserver -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/build/Dockerfile.etcd b/build/Dockerfile.etcd index 2f5fb1d66..dc280cd99 100644 --- a/build/Dockerfile.etcd +++ b/build/Dockerfile.etcd @@ -13,15 +13,17 @@ # limitations under the License. ARG ALPINE_VERSION=latest +ARG GOLANG_BOOKWORM_VERSION=1.25.7-bookworm +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:1.25.7-bookworm AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_BOOKWORM_VERSION} AS builder WORKDIR /workspace ARG ETCD_VER=v3.5.21 RUN curl -L https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz \ | tar --strip-components 1 -xvz "etcd-${ETCD_VER}-linux-amd64/etcd" -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/build/Makefile b/build/Makefile index 39b27c160..f59f7ef60 100644 --- a/build/Makefile +++ b/build/Makefile @@ -20,8 +20,16 @@ ALPINE_VERSION ?= 3.23.3 .PHONY: build-image docker-build build-image docker-build: - docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f ./Dockerfile "$(PORCHDIR)" + docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f ./Dockerfile "$(PORCHDIR)" .PHONY: push-image docker-push push-image docker-push: - docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f ./Dockerfile "$(PORCHDIR)" + docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f ./Dockerfile "$(PORCHDIR)" diff --git a/controllers/Dockerfile b/controllers/Dockerfile index b6681d85c..bbf47c8c9 100644 --- a/controllers/Dockerfile +++ b/controllers/Dockerfile @@ -14,8 +14,9 @@ ARG ALPINE_VERSION=latest ARG GOLANG_BOOKWORM_VERSION=latest +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:${GOLANG_BOOKWORM_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_BOOKWORM_VERSION} AS builder WORKDIR /go/src @@ -42,7 +43,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /porch-controllers -v . -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/controllers/Makefile b/controllers/Makefile index 12df525f0..36cc81081 100644 --- a/controllers/Makefile +++ b/controllers/Makefile @@ -18,8 +18,16 @@ IMAGE_NAME ?= porch-controllers .PHONY: build-image docker-build build-image docker-build: - docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f Dockerfile .. + docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f Dockerfile .. .PHONY: push-image docker-push push-image docker-push: - docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f Dockerfile .. + docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f Dockerfile .. diff --git a/func/Dockerfile b/func/Dockerfile index cf71fa1d6..12857a12f 100644 --- a/func/Dockerfile +++ b/func/Dockerfile @@ -14,20 +14,22 @@ ARG ALPINE_VERSION=latest ARG GOLANG_ALPINE_VERSION=latest +ARG PORCH_GHCR_PREFIX_URL=ghcr.io/kptdev/krm-functions-catalog +ARG DOCKERHUB_MIRROR=docker.io -FROM ghcr.io/kptdev/krm-functions-catalog/apply-setters:v0.2.4 AS apply-setters -FROM ghcr.io/kptdev/krm-functions-catalog/ensure-name-substring:v0.2.1 AS ensure-name-substring -FROM ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2.3 AS search-replace -FROM ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.7 AS set-annotations -FROM ghcr.io/kptdev/krm-functions-catalog/set-image:v0.2.2 AS set-image -FROM ghcr.io/kptdev/krm-functions-catalog/set-labels:v0.2.4 AS set-labels -FROM ghcr.io/kptdev/krm-functions-catalog/set-namespace:v0.4.5 AS set-namespace -FROM ghcr.io/kptdev/krm-functions-catalog/set-project-id:v0.2.1 AS set-project-id -FROM ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 AS starlark -FROM ghcr.io/kptdev/krm-functions-catalog/upsert-resource:v0.2.3 AS upsert-resource +FROM ${PORCH_GHCR_PREFIX_URL}/apply-setters:v0.2.4 AS apply-setters +FROM ${PORCH_GHCR_PREFIX_URL}/ensure-name-substring:v0.2.1 AS ensure-name-substring +FROM ${PORCH_GHCR_PREFIX_URL}/search-replace:v0.2.3 AS search-replace +FROM ${PORCH_GHCR_PREFIX_URL}/set-annotations:v0.1.7 AS set-annotations +FROM ${PORCH_GHCR_PREFIX_URL}/set-image:v0.2.2 AS set-image +FROM ${PORCH_GHCR_PREFIX_URL}/set-labels:v0.2.4 AS set-labels +FROM ${PORCH_GHCR_PREFIX_URL}/set-namespace:v0.4.5 AS set-namespace +FROM ${PORCH_GHCR_PREFIX_URL}/set-project-id:v0.2.1 AS set-project-id +FROM ${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5 AS starlark +FROM ${PORCH_GHCR_PREFIX_URL}/upsert-resource:v0.2.3 AS upsert-resource -FROM golang:${GOLANG_ALPINE_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_ALPINE_VERSION} AS builder WORKDIR /go/src RUN go install github.com/grpc-ecosystem/grpc-health-probe@v0.4.11 @@ -49,7 +51,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ cd func ; go build -trimpath -ldflags="-s -w" -v -o /server ./server -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/func/Dockerfile-wrapperserver b/func/Dockerfile-wrapperserver index 538692b76..f96e3ffeb 100644 --- a/func/Dockerfile-wrapperserver +++ b/func/Dockerfile-wrapperserver @@ -14,8 +14,9 @@ ARG ALPINE_VERSION=latest ARG GOLANG_ALPINE_VERSION=latest +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:${GOLANG_ALPINE_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_ALPINE_VERSION} AS builder WORKDIR /go/src @@ -39,7 +40,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ cd func ; go build -trimpath -ldflags="-s -w" -v -o /wrapper-server/wrapper-server ./wrapper-server RUN cp $GOPATH/bin/grpc-health-probe /wrapper-server/ -FROM alpine:${ALPINE_VERSION} +FROM ${DOCKERHUB_MIRROR}/alpine:${ALPINE_VERSION} RUN addgroup -g 10001 nonroot && \ adduser -D -G nonroot -u 10001 nonroot diff --git a/func/Makefile b/func/Makefile index 4df1bd297..01116b6dd 100644 --- a/func/Makefile +++ b/func/Makefile @@ -32,10 +32,30 @@ $(COMPILED_PROTO): evaluator/evaluator.proto .PHONY: build-image docker-build build-image docker-build: - docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) -f ./Dockerfile "$(PORCHDIR)" - docker buildx build --load --tag $(IMAGE_REPO)/$(WRAPPER_SERVER_IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) -f ./Dockerfile-wrapperserver "$(PORCHDIR)" + docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + $$(if [ -n "${PORCH_GHCR_PREFIX_URL}" ]; then echo "--build-arg PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL}"; fi) \ + -f ./Dockerfile "$(PORCHDIR)" + docker buildx build --load --tag $(IMAGE_REPO)/$(WRAPPER_SERVER_IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + $$(if [ -n "${PORCH_GHCR_PREFIX_URL}" ]; then echo "--build-arg PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL}"; fi) \ + -f ./Dockerfile-wrapperserver "$(PORCHDIR)" .PHONY: push-image docker-push push-image docker-push: - docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) -f ./Dockerfile "$(PORCHDIR)" - docker buildx build --push --tag $(IMAGE_REPO)/$(WRAPPER_SERVER_IMAGE_NAME):$(IMAGE_TAG) --build-arg ALPINE_VERSION=$(ALPINE_VERSION) --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) -f ./Dockerfile-wrapperserver "$(PORCHDIR)" + docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + $$(if [ -n "${PORCH_GHCR_PREFIX_URL}" ]; then echo "--build-arg PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL}"; fi) \ + -f ./Dockerfile "$(PORCHDIR)" + docker buildx build --push --tag $(IMAGE_REPO)/$(WRAPPER_SERVER_IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg ALPINE_VERSION=$(ALPINE_VERSION) \ + --build-arg GOLANG_ALPINE_VERSION=$(GOLANG_ALPINE_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + $$(if [ -n "${PORCH_GHCR_PREFIX_URL}" ]; then echo "--build-arg PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL}"; fi) \ + -f ./Dockerfile-wrapperserver "$(PORCHDIR)" diff --git a/go.mod b/go.mod index bcca49e43..0ee342396 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/google/go-containerregistry v0.20.6 github.com/google/uuid v1.6.0 github.com/jackc/pgx/v5 v5.9.2 + github.com/joho/godotenv v1.5.1 github.com/kptdev/kpt v1.0.0-beta.62.1 github.com/kptdev/krm-functions-catalog/functions/go/apply-replacements v0.1.5 github.com/kptdev/krm-functions-catalog/functions/go/apply-setters v0.2.4 diff --git a/go.sum b/go.sum index 7650f65f6..3d4acca29 100644 --- a/go.sum +++ b/go.sum @@ -246,6 +246,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= diff --git a/make/build.mk b/make/build.mk index 6964a11af..7d73d4209 100644 --- a/make/build.mk +++ b/make/build.mk @@ -51,21 +51,21 @@ porchctl: .PHONY: build-images build-images: - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ build-image - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ build-image - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_ALPINE_VERSION="$(GOLANG_ALPINE_VERSION)" IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ build-image - GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ build-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ build-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ build-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_ALPINE_VERSION="$(GOLANG_ALPINE_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ build-image + GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ build-image .PHONY: push-images push-images: - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ push-image - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ push-image - ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_ALPINE_VERSION="$(GOLANG_ALPINE_VERSION)" IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ push-image - GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ push-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ push-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_CONTROLLERS_IMAGE)" make -C controllers/ push-image + ALPINE_VERSION="$(ALPINE_VERSION)" GOLANG_ALPINE_VERSION="$(GOLANG_ALPINE_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_FUNCTION_RUNNER_IMAGE)" WRAPPER_SERVER_IMAGE_NAME="$(PORCH_WRAPPER_SERVER_IMAGE)" make -C func/ push-image + GOLANG_BOOKWORM_VERSION="$(GOLANG_BOOKWORM_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(TEST_GIT_SERVER_IMAGE)" make -C test/ push-image .PHONY: dev-server dev-server: - ALPINE_VERSION="$(ALPINE_VERSION)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ push-image + ALPINE_VERSION="$(ALPINE_VERSION)" DOCKERHUB_MIRROR="$(DOCKERHUB_MIRROR)" IMAGE_NAME="$(PORCH_SERVER_IMAGE)" make -C build/ push-image kubectl set image -n porch-system deployment/porch-server porch-server=$(IMAGE_REPO)/$(PORCH_SERVER_IMAGE):${IMAGE_TAG} .PHONY: update-kube-apiserver-vendoring diff --git a/make/deploy.mk b/make/deploy.mk index 6fc7fea6b..12d0f95a3 100644 --- a/make/deploy.mk +++ b/make/deploy.mk @@ -166,7 +166,7 @@ deploy-gitea-dev-pkg:## Deploy gitea development package --kubeconfig $(KUBECONFIG) .PHONY: setup-dev-env -setup-dev-env: PORCH_TEST_CLUSTER=porch-test ## Setup gitea, Metallb and test repository in kind cluster +setup-dev-env: PORCH_TEST_CLUSTER=porch-test setup-dev-env: GIT_REPO_NAME=porch-test -setup-dev-env: +setup-dev-env: ## Setup gitea, Metallb and test repository in kind cluster ./scripts/setup-dev-env.sh diff --git a/pkg/cache/dbcache/dbcache_test.go b/pkg/cache/dbcache/dbcache_test.go index 42024d7ae..273894317 100644 --- a/pkg/cache/dbcache/dbcache_test.go +++ b/pkg/cache/dbcache/dbcache_test.go @@ -23,6 +23,7 @@ import ( "time" embeddedpostgres "github.com/fergusstrange/embedded-postgres" + "github.com/joho/godotenv" configapi "github.com/kptdev/porch/api/porchconfig/v1alpha1" "github.com/kptdev/porch/pkg/cache/testutil" cachetypes "github.com/kptdev/porch/pkg/cache/types" @@ -38,6 +39,7 @@ import ( ) const defaultPorchSQLSchema = "api/sql/porch-db.sql" +const dbMavenMirrorEnv = "DB_MAVEN_MIRROR" type DbTestSuite struct { suite.Suite @@ -61,12 +63,21 @@ func (t *DbTestSuite) Context() context.Context { } func (t *DbTestSuite) SetupSuite() { - postgres := embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig(). + if err := godotenv.Load("../../../.env"); err != nil { + t.T().Logf("Failed to load .env file: %v", err) + } + + config := embeddedpostgres.DefaultConfig(). Username("porch"). Password("porch"). Database("porch"). - Port(55432)) + Port(55432) + + if mavenMirror := os.Getenv(dbMavenMirrorEnv); mavenMirror != "" { + config = config.BinaryRepositoryURL(mavenMirror) + } + postgres := embeddedpostgres.NewDatabase(config) err := postgres.Start() t.Require().NoError(err, "could not start test instance of postgres") diff --git a/scripts/common.sh b/scripts/common.sh index a7824778d..3a24ae591 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -2,7 +2,34 @@ # Fallback defaults for direct script execution # NOTE: Users should use root Makefile targets instead of calling scripts directly -# Only set defaults if variables are not already exported from Makefile +# Only set defaults if variables are not already exported from Makefile. +# .env entries do not override variables already set in the environment. + +PORCHDIR=${PORCHDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)} + +if [[ -f "${PORCHDIR}/.env" ]]; then + _env_trim() { + local s="$1" + s="${s#"${s%%[![:space:]]*}"}" + s="${s%"${s##*[![:space:]]}"}" + printf '%s' "$s" + } + while IFS= read -r line || [[ -n "$line" ]]; do + [[ "$line" =~ ^[[:space:]]*# ]] && continue + [[ -z "${line//[[:space:]]/}" ]] && continue + [[ "$line" != *=* ]] && continue + key="${line%%=*}" + value="${line#*=}" + key="$(_env_trim "$key")" + value="$(_env_trim "$value")" + [[ -z "$key" ]] && continue + [[ -n "${!key+x}" ]] && continue + export "${key}=${value}" + done < "${PORCHDIR}/.env" + unset -f _env_trim + unset -v key value line +fi + IMAGE_REPO=${IMAGE_REPO:-ghcr.io/kptdev} PORCH_SERVER_IMAGE=${PORCH_SERVER_IMAGE:-porch-server} @@ -15,7 +42,6 @@ SKIP_IMG_BUILD=${SKIP_IMG_BUILD:-false} SKIP_PORCHSERVER_BUILD=${SKIP_PORCHSERVER_BUILD:-false} SKIP_CONTROLLER_BUILD=${SKIP_CONTROLLER_BUILD:-false} -PORCHDIR=${PORCHDIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)} KIND_CONTEXT_NAME=${KIND_CONTEXT_NAME:-porch-test} ENABLED_RECONCILERS=${ENABLED_RECONCILERS:-"packagevariants,packagevariantsets,repositories"} PORCH_CACHE_TYPE=${PORCH_CACHE_TYPE:-CR} @@ -23,3 +49,8 @@ FN_RUNNER_WARM_UP_POD_CACHE=${FN_RUNNER_WARM_UP_POD_CACHE:-true} DB_PUSH_DRAFTS_TO_GIT=${DB_PUSH_DRAFTS_TO_GIT:-false} CREATE_V1ALPHA2_RPKG=${CREATE_V1ALPHA2_RPKG:-false} DEPLOYPORCHCONFIGDIR=${DEPLOYPORCHCONFIGDIR:-${PORCHDIR}/.build/deploy} + +DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR:-""} +DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR%/} # remove '/' suffix +PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL:-ghcr.io/kptdev/krm-functions-catalog} +PORCH_GHCR_PREFIX_URL=${PORCH_GHCR_PREFIX_URL%/} # remove '/' suffix diff --git a/scripts/create-deployment-blueprint.sh b/scripts/create-deployment-blueprint.sh index 4090d0a30..5a1a7d22b 100755 --- a/scripts/create-deployment-blueprint.sh +++ b/scripts/create-deployment-blueprint.sh @@ -18,10 +18,13 @@ set -e # Exit on error set -u # Must predefine variables set -o pipefail # Check errors in piped commands +# Source common configuration +source "$(dirname "$0")/common.sh" + PORCH_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -STARLARK_IMG="ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5" -SEARCH_REPLACE_IMG="ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2" -SET_IMAGE_IMG="ghcr.io/kptdev/krm-functions-catalog/set-image:v0.2.2" +STARLARK_IMG="${PORCH_GHCR_PREFIX_URL}/starlark:v0.5" +SEARCH_REPLACE_IMG="${PORCH_GHCR_PREFIX_URL}/search-replace:v0.2" +SET_IMAGE_IMG="${PORCH_GHCR_PREFIX_URL}/set-image:v0.2.2" function error() { cat </dev/null 2>&1 & PORT_FORWARD_PID=$! sleep 3 - # Delete existing repo h1 "Deleting existing test-blueprints repository" curl -s -X DELETE "http://porch:secret@localhost:3000/api/v1/repos/porch/test-blueprints" >/dev/null 2>&1 || true - # Recreate repo from bundle h1 "Recreating test-blueprints repository" curl -s -H "content-type: application/json" "http://porch:secret@localhost:3000/api/v1/user/repos" --data '{"name":"test-blueprints"}' >/dev/null 2>&1 - TEST_BLUEPRINTS_TMP_DIR=$(mktemp -d) cd "$TEST_BLUEPRINTS_TMP_DIR" h1 "Cloning from bundle: $TEST_BLUEPRINTS_PATH" git clone "$TEST_BLUEPRINTS_PATH" -b main cd test-blueprints - git gc git remote rename origin upstream git remote add origin "http://porch:secret@localhost:3000/porch/test-blueprints" git push -u origin --all git push -u origin --tags - cd "${git_root}" rm -fr "$TEST_BLUEPRINTS_TMP_DIR" - kill $PORT_FORWARD_PID 2>/dev/null || true echo "Test-blueprints repo reloaded from bundle" } @@ -80,7 +76,6 @@ function retry_curl() { local delay=$2 shift 2 local attempt=1 - while [ $attempt -le $max_attempts ]; do if "$@"; then return 0 @@ -89,7 +84,6 @@ function retry_curl() { sleep $delay attempt=$((attempt + 1)) done - echo "ERROR: Command failed after $max_attempts attempts: $*" return 1 } @@ -97,12 +91,14 @@ function retry_curl() { h1 Install Gitea and init test repos mkdir -p "${git_root}/.build" cd "${git_root}/.build" + +# Always clean and rebuild the package if [ -d gitea ]; then - h1 Dev Gitea pkg already present -else - cp -r "${git_root}/test/pkgs/gitea-dev" gitea + rm -rf gitea fi - + +cp -r "${git_root}/test/pkgs/gitea-dev" gitea + cd "${git_root}/.build/gitea" # Check if the gitea service of type LoadBalancer exists in the 'gitea' namespace @@ -124,7 +120,7 @@ else # Only pin a specific IP if one was explicitly provided; otherwise let MetalLB assign from pool if [[ -n "${gitea_ip}" ]]; then kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.7 \ + --image "${PORCH_GHCR_PREFIX_URL}/set-annotations:v0.1.7" \ --match-kind Service \ --match-name gitea-lb \ --match-namespace gitea \ @@ -133,6 +129,13 @@ else fi kpt fn render + +if [[ -n "${DOCKERHUB_MIRROR}" ]]; then + kpt fn eval . --image "${PORCH_GHCR_PREFIX_URL}/set-image:v0.2.2" -- \ + "name=gitea/gitea" \ + "newName=${DOCKERHUB_MIRROR}/gitea/gitea" +fi + kpt live init || true kpt live apply --inventory-policy=adopt echo "Waiting for gitea deployment to become ready..." diff --git a/scripts/modify-gitea-test-blueprints.sh b/scripts/modify-gitea-test-blueprints.sh index e2d4f0b60..15ed26320 100755 --- a/scripts/modify-gitea-test-blueprints.sh +++ b/scripts/modify-gitea-test-blueprints.sh @@ -20,6 +20,9 @@ set -e # Exit on error set -u # Must predefine variables set -o pipefail # Check errors in piped commands +# Source common configuration +source "$(dirname "$0")/common.sh" + SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" BUNDLE_PATH="$SCRIPT_DIR/../test/pkgs/test-pkgs/test-blueprints.bundle" diff --git a/scripts/remove-controller-from-deployment-config.sh b/scripts/remove-controller-from-deployment-config.sh index e9b5ff517..830bfaa61 100755 --- a/scripts/remove-controller-from-deployment-config.sh +++ b/scripts/remove-controller-from-deployment-config.sh @@ -18,6 +18,9 @@ set -e # Exit on error set -u # Must predefine variables set -o pipefail # Check errors in piped commands +# Source common configuration +source "$(dirname "$0")/common.sh" + self_dir="$(dirname "$(readlink -f "$0")")" git_root="$(readlink -f "${self_dir}/..")" source "${git_root}/scripts/get-kind-metallb-subnet.sh" @@ -43,7 +46,7 @@ cd "${deployment_config_dir}" # expose function-runner to local processes kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Service \ --match-name function-runner \ --match-namespace porch-system \ @@ -56,7 +59,7 @@ for resource in ctx.resource_list["items"]: # remove porch-controllers Deployment from package kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Deployment \ --match-name porch-controllers \ --match-namespace porch-system \ diff --git a/scripts/remove-porch-server-from-deployment-config.sh b/scripts/remove-porch-server-from-deployment-config.sh index 1f7b85e41..e308dad34 100755 --- a/scripts/remove-porch-server-from-deployment-config.sh +++ b/scripts/remove-porch-server-from-deployment-config.sh @@ -17,6 +17,10 @@ set -e # Exit on error set -u # Must predefine variables set -o pipefail # Check errors in piped commands + +# Source common configuration +source "$(dirname "$0")/common.sh" + self_dir="$(dirname "$(readlink -f "$0")")" git_root="$(readlink -f "${self_dir}/..")" @@ -40,7 +44,7 @@ cd "${deployment_config_dir}" # expose function-runner to local processes kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Service \ --match-name function-runner \ --match-namespace porch-system \ @@ -53,7 +57,7 @@ for resource in ctx.resource_list["items"]: # remove porch-server Deployment from package kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Deployment \ --match-name porch-server \ --match-namespace porch-system \ @@ -64,7 +68,7 @@ if [[ "$(uname)" == "Darwin" || -n "${DOCKER_HOST+x}" ]] || docker info 2>/dev/n then echo "--- Docker Desktop detected. ---" kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Service \ --match-name api \ --match-namespace porch-system \ @@ -76,7 +80,7 @@ for resource in ctx.resource_list["items"]: } ' kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2.3 \ + --image "${PORCH_GHCR_PREFIX_URL}/search-replace:v0.2.3" \ --match-kind APIService \ --match-name v1alpha1.porch.kpt.dev \ -- 'by-path=spec.service.port' "put-value=4443" @@ -84,16 +88,16 @@ else echo "--- Local Docker daemon detected. ---" docker_bridge_ip="$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')" kpt fn eval \ - --image upsert-resource:v0.2.0 \ + --image "${PORCH_GHCR_PREFIX_URL}/upsert-resource:v0.2.0" \ --fn-config "${git_root}/deployments/local/porch-api-endpoints.yaml" kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2.3 \ + --image "${PORCH_GHCR_PREFIX_URL}/search-replace:v0.2.3" \ --match-kind Endpoints \ --match-name api \ --match-namespace porch-system \ -- 'by-path=subsets[0].addresses[0].ip' "put-value=$docker_bridge_ip" kpt fn eval \ - --image ghcr.io/kptdev/krm-functions-catalog/starlark:v0.5.5 \ + --image "${PORCH_GHCR_PREFIX_URL}/starlark:v0.5.5" \ --match-kind Service \ --match-name api \ --match-namespace porch-system \ diff --git a/scripts/run-load-test.sh b/scripts/run-load-test.sh index e3d84c6b3..22f4def66 100755 --- a/scripts/run-load-test.sh +++ b/scripts/run-load-test.sh @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Source common configuration +source "$(dirname "$0")/common.sh" + script_name=$(basename "$0") git_repo_server="" @@ -137,9 +140,9 @@ info: pipeline: mutators: - - image: ghcr.io/kptdev/krm-functions-catalog/apply-replacements:v0.1.5 + - image: ${PORCH_GHCR_PREFIX_URL}/apply-replacements:v0.1.5 configPath: apply-replacements-annotation1.yaml - - image: ghcr.io/kptdev/krm-functions-catalog/apply-replacements:v0.1.5 + - image: ${PORCH_GHCR_PREFIX_URL}/apply-replacements:v0.1.5 configPath: apply-replacements-annotation2.yaml EOF diff --git a/test/Dockerfile b/test/Dockerfile index 2ea856113..0dcdb6e85 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: do we still use this? ARG GOLANG_BOOKWORM_VERSION=latest +ARG DOCKERHUB_MIRROR=docker.io -FROM golang:${GOLANG_BOOKWORM_VERSION} AS builder +FROM ${DOCKERHUB_MIRROR}/golang:${GOLANG_BOOKWORM_VERSION} AS builder WORKDIR /go/src @@ -27,7 +29,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ cd test; go build -trimpath -ldflags="-s -w" -v -o /git-server ./git -FROM debian:bookworm-slim +FROM ${DOCKERHUB_MIRROR}/debian:bookworm-slim RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt && rm -rf /var/cache/apt COPY --from=builder /git-server /git-server ENTRYPOINT ["/git-server", "--port=8080"] diff --git a/test/Makefile b/test/Makefile index 8914c7d09..00754e906 100644 --- a/test/Makefile +++ b/test/Makefile @@ -12,14 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO: do we still use this? + IMAGE_TAG ?= latest IMAGE_REPO ?= ghcr.io/kptdev IMAGE_NAME ?= test-git-server .PHONY: build-image docker-build build-image docker-build: - docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f Dockerfile .. + docker buildx build --load --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f Dockerfile .. .PHONY: push-image docker-push push-image docker-push: - docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) -f Dockerfile .. + docker buildx build --push --tag $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_TAG) \ + --build-arg GOLANG_BOOKWORM_VERSION=$(GOLANG_BOOKWORM_VERSION) \ + $$(if [ -n "${DOCKERHUB_MIRROR}" ]; then echo "--build-arg DOCKERHUB_MIRROR=${DOCKERHUB_MIRROR}"; fi) \ + -f Dockerfile ..