From 19de7d5ce1e99a7269b248483801c810afd2145a Mon Sep 17 00:00:00 2001 From: Aryan Puttur Date: Tue, 23 Jun 2026 12:15:57 -0400 Subject: [PATCH 1/3] Add envtest integration tests for kube controller Fixes #2486 Depends on #2506 (NewClientFromRestConfig) for compilation. --- go.mod | 12 +- go.sum | 24 ++++ tests/Makefile | 9 ++ tests/integration/README.md | 68 +++++++++ .../kube/controller/helpers_test.go | 50 +++++++ .../kube/controller/listener_test.go | 64 +++++++++ .../integration/kube/controller/site_test.go | 50 +++++++ .../integration/kube/controller/suite_test.go | 134 ++++++++++++++++++ 8 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 tests/integration/README.md create mode 100644 tests/integration/kube/controller/helpers_test.go create mode 100644 tests/integration/kube/controller/listener_test.go create mode 100644 tests/integration/kube/controller/site_test.go create mode 100644 tests/integration/kube/controller/suite_test.go diff --git a/go.mod b/go.mod index 467d7eef7..f92390806 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/prometheus/client_golang v1.22.0 github.com/skupperproject/skupper-libpod/v4 v4.0.3-0 github.com/spf13/cobra v1.8.1 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 golang.org/x/sync v0.20.0 golang.org/x/sys v0.43.0 golang.org/x/text v0.36.0 @@ -49,17 +49,20 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.7.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/getkin/kin-openapi v0.132.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/analysis v0.21.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -94,9 +97,12 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/speakeasy-api/jsonpath v0.6.0 // indirect github.com/speakeasy-api/openapi-overlay v0.10.2 // indirect + github.com/spf13/afero v1.12.0 // indirect github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect github.com/x448/float16 v0.8.4 // indirect go.mongodb.org/mongo-driver v1.10.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.34.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect @@ -111,7 +117,11 @@ require ( k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect + sigs.k8s.io/controller-runtime v0.21.0 // indirect + sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20250626154428-7fd020cb5fc3 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect ) + +tool sigs.k8s.io/controller-runtime/tools/setup-envtest diff --git a/go.sum b/go.sum index ef884f476..ba232fc78 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= @@ -95,7 +97,10 @@ github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -120,6 +125,8 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/analysis v0.21.2 h1:hXFrOYFHUAMQdu6zwAiKKJHJQ8kqZs1ux/ru1P1wLJU= github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= @@ -244,6 +251,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -363,6 +371,8 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -371,6 +381,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= github.com/openshift/api v0.0.0-20210105115604-44119421ec6b/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg= github.com/openshift/api v0.0.0-20210428205234-a8389931bee7 h1:kYbp8I2qi3bAyHjTj50Lb1GC2ck7SnZX5M/ZYvF3eLI= github.com/openshift/api v0.0.0-20210428205234-a8389931bee7/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg= @@ -417,6 +429,8 @@ github.com/speakeasy-api/jsonpath v0.6.0/go.mod h1:ymb2iSkyOycmzKwbEAYPJV/yi2rSm github.com/speakeasy-api/openapi-overlay v0.10.2 h1:VOdQ03eGKeiHnpb1boZCGm7x8Haj6gST0P3SGTX95GU= github.com/speakeasy-api/openapi-overlay v0.10.2/go.mod h1:n0iOU7AqKpNFfEt6tq7qYITC4f0yzVVdFw0S7hukemg= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= @@ -424,6 +438,8 @@ github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzu github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -470,6 +486,10 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -801,6 +821,10 @@ k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= +sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20250626154428-7fd020cb5fc3 h1:CPPRG6M4YrrYmvtJ1CTJtXvj21SYwpU3NMMX1JCEATg= +sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20250626154428-7fd020cb5fc3/go.mod h1:zxemiV1fQ3IJnXX0VeOb6qbHp2Wtv3s1w+gftNeHWTg= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= diff --git a/tests/Makefile b/tests/Makefile index 82ad56400..804ae13e2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,7 @@ ROOT_PATH := $(shell pwd) +SKUPPER_ROOT := $(abspath $(ROOT_PATH)/..) +TESTFLAGS ?= -v -race -short +ENVTEST_K8S_VERSION ?= 1.33.0 EXTRA_VARS := --extra-vars "@$(ROOT_PATH)/vars.yml" COLLECTION_PATH := $(ROOT_PATH)/e2e/collections/ansible_collections/e2e/tests # Remove expose-pods-by-name from the CI tests, until we close @@ -110,3 +113,9 @@ test-subset: create-venv # Run a subset of tests (comma-separated list) for CI ci-tests: TESTS=$(TESTS_CI) ci-tests: test-subset + +.PHONY: test-integration +test-integration: + @assets=$$(cd $(SKUPPER_ROOT) && go tool setup-envtest use $(ENVTEST_K8S_VERSION) -p path); \ + export KUBEBUILDER_ASSETS="$$assets"; \ + cd $(SKUPPER_ROOT) && go test -tags=integration $(TESTFLAGS) ./tests/integration/kube/controller/... diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 000000000..3805698c5 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,68 @@ +# Integration tests + +Go integration tests that run Skupper components against a real Kubernetes API server +using [envtest](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/envtest) (kube-apiserver + +etcd), without needing a full cluster. + +These sit between unit tests (fake clients, synchronous event processing) and Ansible E2E +tests under `tests/e2e/` (real clusters, cross-site networking). + +## Layout + +Mirrors `internal/` so kube and nonkube integration tests can live alongside their +production packages. + +| Directory | Tests | +|-----------|-------| +| `kube/controller/` | Skupper kube controller (`internal/kube/controller`, `cmd/controller`) | +| `nonkube/controller/` | (future) nonkube controller (`internal/nonkube/controller`) | + +## Prerequisites + +The `setup-envtest` version is pinned in the root `go.mod` `tool` directive (matching +controller-runtime release-0.21 / k8s 1.33). Run from the repository root: + +```bash +go tool setup-envtest use -i 1.33.0 +``` + +Or let `make test-integration` download binaries on first run. + +To pre-download Kubernetes test binaries without running tests: + +```bash +go tool setup-envtest use -i 1.33.0 +``` + + +## Run + +From the repository root: + +```bash +make -C tests test-integration +``` + +Or from `tests/`: + +```bash +make test-integration +``` + +Or directly: + +```bash +export KUBEBUILDER_ASSETS=$(go tool setup-envtest use 1.33.0 -p path) +go test -tags=integration -v ./tests/integration/kube/controller/... +``` + +Default `make test` does **not** run these (they use the `integration` build tag and take +~1 minute). + +## Notes + +- Tests start a shared controller instance and a fresh envtest apiserver per package run. +- Gateway, Contour, and OpenShift Route CRDs are not installed; related watcher errors in + logs are expected and harmless for current scenarios. +- A teardown warning about kube-apiserver shutdown may appear after tests pass; this is a + known envtest quirk and does not indicate test failure. diff --git a/tests/integration/kube/controller/helpers_test.go b/tests/integration/kube/controller/helpers_test.go new file mode 100644 index 000000000..2bda54ac3 --- /dev/null +++ b/tests/integration/kube/controller/helpers_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package kubecontrollertest + +import ( + "testing" + "time" + + "github.com/skupperproject/skupper/internal/fixtures" + "github.com/skupperproject/skupper/internal/utils" + "gotest.tools/v3/assert" + "k8s.io/apimachinery/pkg/api/meta" + + skupperv2alpha1 "github.com/skupperproject/skupper/pkg/apis/skupper/v2alpha1" +) + +func waitFor(t *testing.T, timeout, interval time.Duration, fn func() (bool, error)) { + t.Helper() + err := utils.Retry(interval, int(timeout/interval), fn) + assert.NilError(t, err) +} + +func listenerWithHostPort(name, namespace, host string, port int) *skupperv2alpha1.Listener { + l := fixtures.Listener(name, namespace) + l.Spec.Host = host + l.Spec.Port = port + return l +} + +func routerSelector() map[string]string { + return map[string]string{ + "skupper.io/component": "router", + "application": "skupper-router", + } +} + +func verifyStatus(t *testing.T, expected, actual skupperv2alpha1.Status) { + t.Helper() + assert.Equal(t, expected.StatusType, actual.StatusType, actual.Message) + assert.Equal(t, expected.Message, actual.Message) + for _, c := range expected.Conditions { + existing := meta.FindStatusCondition(actual.Conditions, c.Type) + assert.Assert(t, existing != nil) + assert.Equal(t, c.Status, existing.Status) + assert.Equal(t, c.Reason, existing.Reason) + if c.Message != "" { + assert.Equal(t, c.Message, existing.Message) + } + } +} diff --git a/tests/integration/kube/controller/listener_test.go b/tests/integration/kube/controller/listener_test.go new file mode 100644 index 000000000..90def6c36 --- /dev/null +++ b/tests/integration/kube/controller/listener_test.go @@ -0,0 +1,64 @@ +//go:build integration + +package kubecontrollertest + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/skupperproject/skupper/api/types" + "github.com/skupperproject/skupper/internal/fixtures" + "gotest.tools/v3/assert" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + skupperv2alpha1 "github.com/skupperproject/skupper/pkg/apis/skupper/v2alpha1" +) + +func TestSiteWithListener(t *testing.T) { + tc := setup(t) + namespace := "site-with-listener" + tc.createNamespace(namespace) + + ctx := context.Background() + _, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Create(ctx, fixtures.Site("mysvc", namespace), metav1.CreateOptions{}) + assert.NilError(t, err) + _, err = tc.clients.GetSkupperClient().SkupperV2alpha1().Listeners(namespace).Create(ctx, listenerWithHostPort("mylistener", namespace, "mysvc", 8080), metav1.CreateOptions{}) + assert.NilError(t, err) + + waitFor(t, 30*time.Second, 250*time.Millisecond, func() (bool, error) { + l, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Listeners(namespace).Get(ctx, "mylistener", metav1.GetOptions{}) + if err != nil { + return false, nil + } + configured := meta.FindStatusCondition(l.Status.Conditions, skupperv2alpha1.CONDITION_TYPE_CONFIGURED) + if configured == nil || configured.Status != metav1.ConditionTrue { + return false, nil + } + _, err = tc.clients.GetKubeClient().CoreV1().Services(namespace).Get(ctx, "mysvc", metav1.GetOptions{}) + return err == nil, nil + }) + + actualSite, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Get(ctx, "mysvc", metav1.GetOptions{}) + assert.NilError(t, err) + verifyStatus(t, fixtures.Status(skupperv2alpha1.StatusPending, "Not Running", + fixtures.Condition(skupperv2alpha1.CONDITION_TYPE_CONFIGURED, metav1.ConditionTrue, "Ready", "OK")), + actualSite.Status.Status) + + deployment, err := tc.clients.GetKubeClient().AppsV1().Deployments(namespace).Get(ctx, "skupper-router", metav1.GetOptions{}) + assert.NilError(t, err) + assert.Equal(t, deployment.Labels["skupper.io/component"], "router") + + svc, err := tc.clients.GetKubeClient().CoreV1().Services(namespace).Get(ctx, "mysvc", metav1.GetOptions{}) + assert.NilError(t, err) + assert.DeepEqual(t, svc.Spec.Selector, routerSelector()) + assert.Equal(t, len(svc.Spec.Ports), 1) + assert.Equal(t, svc.Spec.Ports[0].Port, int32(8080)) + assert.Equal(t, svc.Labels["internal.skupper.io/listener"], "mylistener") + + routerConfig, err := tc.clients.GetKubeClient().CoreV1().ConfigMaps(namespace).Get(ctx, "skupper-router", metav1.GetOptions{}) + assert.NilError(t, err) + assert.Assert(t, strings.Contains(routerConfig.Data[types.TransportConfigFile], "listener/mylistener")) +} diff --git a/tests/integration/kube/controller/site_test.go b/tests/integration/kube/controller/site_test.go new file mode 100644 index 000000000..10d2dff03 --- /dev/null +++ b/tests/integration/kube/controller/site_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package kubecontrollertest + +import ( + "context" + "testing" + "time" + + "github.com/skupperproject/skupper/internal/fixtures" + "gotest.tools/v3/assert" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + skupperv2alpha1 "github.com/skupperproject/skupper/pkg/apis/skupper/v2alpha1" +) + +func TestSimpleSite(t *testing.T) { + tc := setup(t) + namespace := "simple-site" + tc.createNamespace(namespace) + + ctx := context.Background() + _, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Create(ctx, fixtures.Site("mysite", namespace), metav1.CreateOptions{}) + assert.NilError(t, err) + + waitFor(t, 30*time.Second, 250*time.Millisecond, func() (bool, error) { + actual, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Get(ctx, "mysite", metav1.GetOptions{}) + if err != nil { + return false, nil + } + configured := meta.FindStatusCondition(actual.Status.Conditions, skupperv2alpha1.CONDITION_TYPE_CONFIGURED) + if configured == nil || configured.Status != metav1.ConditionTrue { + return false, nil + } + return true, nil + }) + + actualSite, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Get(ctx, "mysite", metav1.GetOptions{}) + assert.NilError(t, err) + verifyStatus(t, fixtures.Status(skupperv2alpha1.StatusPending, "Not Running", + fixtures.Condition(skupperv2alpha1.CONDITION_TYPE_CONFIGURED, metav1.ConditionTrue, "Ready", "OK")), + actualSite.Status.Status) + + deployment, err := tc.clients.GetKubeClient().AppsV1().Deployments(namespace).Get(ctx, "skupper-router", metav1.GetOptions{}) + assert.NilError(t, err) + assert.Equal(t, deployment.Labels["skupper.io/component"], "router") + assert.Equal(t, deployment.Labels["application"], "skupper-router") + assert.Equal(t, len(deployment.Spec.Template.Spec.Containers), 2) +} diff --git a/tests/integration/kube/controller/suite_test.go b/tests/integration/kube/controller/suite_test.go new file mode 100644 index 000000000..eb72d76cf --- /dev/null +++ b/tests/integration/kube/controller/suite_test.go @@ -0,0 +1,134 @@ +//go:build integration + +// Integration tests for the Skupper kube controller (internal/kube/controller, cmd/controller). +package kubecontrollertest + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + internalclient "github.com/skupperproject/skupper/internal/kube/client" + kubecontroller "github.com/skupperproject/skupper/internal/kube/controller" + "gotest.tools/v3/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +const controllerInstallNamespace = "skupper-system" + +var ( + envTestConfig *rest.Config + testEnv *envtest.Environment + envTestClients *internalclient.KubeClient + envTestStopCh chan struct{} + envTestStopped chan struct{} +) + +func TestMain(m *testing.M) { + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + ControlPlaneStopTimeout: time.Minute, + } + + var err error + envTestConfig, err = testEnv.Start() + if err != nil { + panic(err) + } + defer func() { + stopSharedController() + if err := testEnv.Stop(); err != nil { + fmt.Fprintf(os.Stderr, "envtest teardown warning: %v\n", err) + } + }() + + if err := startSharedController(); err != nil { + panic(err) + } + + m.Run() +} + +func startSharedController() error { + os.Setenv("NAMESPACE", controllerInstallNamespace) + os.Setenv("CONTROLLER_NAME", "test-controller") + os.Setenv("SKUPPER_METRICS_DISABLE", "true") + + flags := flag.NewFlagSet("integration-test", flag.ContinueOnError) + config, err := kubecontroller.BoundConfig(flags) + if err != nil { + return err + } + + clients, err := internalclient.NewClientFromRestConfig(envTestConfig, controllerInstallNamespace) + if err != nil { + return err + } + envTestClients = clients + + ctx := context.Background() + _, err = clients.GetKubeClient().CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: controllerInstallNamespace}, + }, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return err + } + + ctrl, err := kubecontroller.NewController(clients, config) + if err != nil { + return err + } + + envTestStopCh = make(chan struct{}) + envTestStopped = make(chan struct{}) + go func() { + _ = ctrl.Run(envTestStopCh) + close(envTestStopped) + }() + + time.Sleep(200 * time.Millisecond) + return nil +} + +func stopSharedController() { + if envTestStopCh == nil { + return + } + close(envTestStopCh) + select { + case <-envTestStopped: + case <-time.After(10 * time.Second): + fmt.Fprintf(os.Stderr, "controller did not stop within 10s\n") + } + time.Sleep(3 * time.Second) +} + +type testContext struct { + t *testing.T + clients *internalclient.KubeClient +} + +func setup(t *testing.T) *testContext { + t.Helper() + return &testContext{t: t, clients: envTestClients} +} + +func (tc *testContext) createNamespace(name string) { + tc.t.Helper() + ctx := context.Background() + _, err := tc.clients.GetKubeClient().CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + }, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + assert.NilError(tc.t, err) + } +} From cfeb6ad8357a547aab83abef0393095405788802 Mon Sep 17 00:00:00 2001 From: Aryan Puttur Date: Tue, 23 Jun 2026 13:05:08 -0400 Subject: [PATCH 2/3] Added coderabbit suggested fixes --- tests/Makefile | 4 ++++ tests/integration/README.md | 4 ++-- tests/integration/kube/controller/helpers_test.go | 11 +++++++++++ tests/integration/kube/controller/listener_test.go | 9 ++++++--- tests/integration/kube/controller/site_test.go | 4 ++-- tests/integration/kube/controller/suite_test.go | 14 +++++++++++--- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 804ae13e2..04584f97f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -117,5 +117,9 @@ ci-tests: test-subset .PHONY: test-integration test-integration: @assets=$$(cd $(SKUPPER_ROOT) && go tool setup-envtest use $(ENVTEST_K8S_VERSION) -p path); \ + if [ -z "$$assets" ]; then \ + echo "setup-envtest did not return KUBEBUILDER_ASSETS; run 'go tool setup-envtest use $(ENVTEST_K8S_VERSION)' from the repository root"; \ + exit 1; \ + fi; \ export KUBEBUILDER_ASSETS="$$assets"; \ cd $(SKUPPER_ROOT) && go test -tags=integration $(TESTFLAGS) ./tests/integration/kube/controller/... diff --git a/tests/integration/README.md b/tests/integration/README.md index 3805698c5..6edb44eec 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -23,7 +23,7 @@ The `setup-envtest` version is pinned in the root `go.mod` `tool` directive (mat controller-runtime release-0.21 / k8s 1.33). Run from the repository root: ```bash -go tool setup-envtest use -i 1.33.0 +go tool setup-envtest use 1.33.0 ``` Or let `make test-integration` download binaries on first run. @@ -31,7 +31,7 @@ Or let `make test-integration` download binaries on first run. To pre-download Kubernetes test binaries without running tests: ```bash -go tool setup-envtest use -i 1.33.0 +go tool setup-envtest use 1.33.0 ``` diff --git a/tests/integration/kube/controller/helpers_test.go b/tests/integration/kube/controller/helpers_test.go index 2bda54ac3..50cef4e90 100644 --- a/tests/integration/kube/controller/helpers_test.go +++ b/tests/integration/kube/controller/helpers_test.go @@ -9,6 +9,7 @@ import ( "github.com/skupperproject/skupper/internal/fixtures" "github.com/skupperproject/skupper/internal/utils" "gotest.tools/v3/assert" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" skupperv2alpha1 "github.com/skupperproject/skupper/pkg/apis/skupper/v2alpha1" @@ -20,6 +21,16 @@ func waitFor(t *testing.T, timeout, interval time.Duration, fn func() (bool, err assert.NilError(t, err) } +func retryOnNotFound(err error) (bool, error) { + if err == nil { + return true, nil + } + if errors.IsNotFound(err) { + return false, nil + } + return false, err +} + func listenerWithHostPort(name, namespace, host string, port int) *skupperv2alpha1.Listener { l := fixtures.Listener(name, namespace) l.Spec.Host = host diff --git a/tests/integration/kube/controller/listener_test.go b/tests/integration/kube/controller/listener_test.go index 90def6c36..2d7e8ef35 100644 --- a/tests/integration/kube/controller/listener_test.go +++ b/tests/integration/kube/controller/listener_test.go @@ -30,15 +30,18 @@ func TestSiteWithListener(t *testing.T) { waitFor(t, 30*time.Second, 250*time.Millisecond, func() (bool, error) { l, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Listeners(namespace).Get(ctx, "mylistener", metav1.GetOptions{}) - if err != nil { - return false, nil + if done, err := retryOnNotFound(err); !done { + return false, err } configured := meta.FindStatusCondition(l.Status.Conditions, skupperv2alpha1.CONDITION_TYPE_CONFIGURED) if configured == nil || configured.Status != metav1.ConditionTrue { return false, nil } _, err = tc.clients.GetKubeClient().CoreV1().Services(namespace).Get(ctx, "mysvc", metav1.GetOptions{}) - return err == nil, nil + if done, err := retryOnNotFound(err); !done { + return false, err + } + return true, nil }) actualSite, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Get(ctx, "mysvc", metav1.GetOptions{}) diff --git a/tests/integration/kube/controller/site_test.go b/tests/integration/kube/controller/site_test.go index 10d2dff03..d1913fda4 100644 --- a/tests/integration/kube/controller/site_test.go +++ b/tests/integration/kube/controller/site_test.go @@ -26,8 +26,8 @@ func TestSimpleSite(t *testing.T) { waitFor(t, 30*time.Second, 250*time.Millisecond, func() (bool, error) { actual, err := tc.clients.GetSkupperClient().SkupperV2alpha1().Sites(namespace).Get(ctx, "mysite", metav1.GetOptions{}) - if err != nil { - return false, nil + if done, err := retryOnNotFound(err); !done { + return false, err } configured := meta.FindStatusCondition(actual.Status.Conditions, skupperv2alpha1.CONDITION_TYPE_CONFIGURED) if configured == nil || configured.Status != metav1.ConditionTrue { diff --git a/tests/integration/kube/controller/suite_test.go b/tests/integration/kube/controller/suite_test.go index eb72d76cf..e4926642a 100644 --- a/tests/integration/kube/controller/suite_test.go +++ b/tests/integration/kube/controller/suite_test.go @@ -90,13 +90,21 @@ func startSharedController() error { envTestStopCh = make(chan struct{}) envTestStopped = make(chan struct{}) + runErrCh := make(chan error, 1) go func() { - _ = ctrl.Run(envTestStopCh) + runErrCh <- ctrl.Run(envTestStopCh) close(envTestStopped) }() - time.Sleep(200 * time.Millisecond) - return nil + select { + case err := <-runErrCh: + if err != nil { + return fmt.Errorf("controller failed to start: %w", err) + } + return fmt.Errorf("controller stopped unexpectedly during startup") + case <-time.After(500 * time.Millisecond): + return nil + } } func stopSharedController() { From 5f9067a86e588146c9e0876d4f7da8505b696881 Mon Sep 17 00:00:00 2001 From: Aryan Puttur Date: Tue, 30 Jun 2026 14:10:14 -0400 Subject: [PATCH 3/3] Added documentation in README for USE_EXISTING_CLUSTER --- tests/integration/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/integration/README.md b/tests/integration/README.md index 6edb44eec..0e27d2c33 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -59,6 +59,31 @@ go test -tags=integration -v ./tests/integration/kube/controller/... Default `make test` does **not** run these (they use the `integration` build tag and take ~1 minute). +## Running against an existing cluster + +By default, tests start a local envtest apiserver (kube-apiserver + etcd). To run against a +full Kubernetes cluster instead, set `USE_EXISTING_CLUSTER=true`. envtest will use your +current kubeconfig (`KUBECONFIG` or `~/.kube/config`) and install Skupper CRDs from +`config/crd/bases` before the tests run. + +```bash +# Ensure kubectl context points at the target cluster +kubectl config current-context + +USE_EXISTING_CLUSTER=true make -C tests test-integration +``` + +Or directly: + +```bash +export USE_EXISTING_CLUSTER=true +go test -tags=integration -v ./tests/integration/kube/controller/... +``` + +When using an existing cluster, `setup-envtest` / `KUBEBUILDER_ASSETS` are not required. +The in-process controller still runs locally; tests create namespaces and Skupper resources +on the cluster — use a development or disposable cluster, not production. + ## Notes - Tests start a shared controller instance and a fresh envtest apiserver per package run.