diff --git a/Dockerfile.microshift b/Dockerfile.microshift index b738054ae..f0ca11b3c 100644 --- a/Dockerfile.microshift +++ b/Dockerfile.microshift @@ -5,7 +5,7 @@ WORKDIR /usr/src/multus-cni ENV CGO_ENABLED=1 ENV GO111MODULE=off ENV VERSION=rhel9 COMMIT=unset -RUN ./hack/build-go.sh +RUN GO111MODULE=on go mod vendor && ./hack/build-go.sh WORKDIR / FROM registry.ci.openshift.org/ocp/4.22:base-rhel9 diff --git a/Dockerfile.openshift b/Dockerfile.openshift index 7d34a784e..04236e9b5 100644 --- a/Dockerfile.openshift +++ b/Dockerfile.openshift @@ -7,7 +7,7 @@ WORKDIR /usr/src/multus-cni ENV CGO_ENABLED=1 ENV GO111MODULE=off ENV VERSION=rhel10 COMMIT=unset -RUN ./hack/build-go.sh && \ +RUN GO111MODULE=on go mod vendor && ./hack/build-go.sh && \ cd /usr/src/multus-cni/bin WORKDIR / @@ -17,8 +17,10 @@ WORKDIR /usr/src/multus-cni ENV CGO_ENABLED=1 ENV GO111MODULE=off ENV VERSION=rhel9 COMMIT=unset -RUN ./hack/build-go.sh && \ +RUN GO111MODULE=on go mod vendor && ./hack/build-go.sh && \ cd /usr/src/multus-cni/bin +ENV GO111MODULE=on +RUN make build-e2e-tests && gzip -9 test/bin/multus-cni-tests-ext WORKDIR / FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.25-openshift-4.22 AS rhel8 @@ -27,7 +29,7 @@ WORKDIR /usr/src/multus-cni ENV CGO_ENABLED=1 ENV GO111MODULE=off ENV VERSION=rhel8 COMMIT=unset -RUN ./hack/build-go.sh && \ +RUN GO111MODULE=on go mod vendor && ./hack/build-go.sh && \ cd /usr/src/multus-cni/bin WORKDIR / @@ -40,6 +42,7 @@ RUN dnf install -y util-linux && dnf clean all && \ mkdir -p /usr/src/multus-cni/rhel8/bin COPY --from=rhel10 /usr/src/multus-cni/bin /usr/src/multus-cni/rhel10/bin COPY --from=rhel9 /usr/src/multus-cni/bin /usr/src/multus-cni/rhel9/bin +COPY --from=rhel9 /usr/src/multus-cni/test/bin/multus-cni-tests-ext.gz /usr/bin/multus-cni-tests-ext.gz COPY --from=rhel8 /usr/src/multus-cni/bin /usr/src/multus-cni/rhel8/bin # copy container base image binary to /usr/src/multus-cni/bin RUN bash -c '. /etc/os-release; \ diff --git a/Makefile b/Makefile index c76c55968..369fcbfd3 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,9 @@ build: test: sudo ./hack/test-go.sh + +.PHONY: build-e2e-tests +build-e2e-tests: + @echo "Building multus-cni-tests-ext binary..." + $(MAKE) -C test build \ No newline at end of file diff --git a/go.mod b/go.mod index 83c85aeb2..6b43d701b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,9 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.7.6 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 + github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38 github.com/prometheus/client_golang v1.23.2 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 github.com/vishvananda/netlink v1.3.1 golang.org/x/net v0.48.0 @@ -27,6 +29,7 @@ require ( require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -37,24 +40,31 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/cel-go v0.17.8 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/spdystream v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/vishvananda/netns v0.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.30.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sync v0.19.0 // indirect @@ -62,6 +72,7 @@ require ( golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.9.0 // indirect golang.org/x/tools v0.39.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect @@ -74,3 +85,5 @@ require ( sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) + +replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 diff --git a/go.sum b/go.sum index 791b053c2..4892e1074 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -10,6 +14,7 @@ github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEm github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= github.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6LC3U+QrEANuQ= github.com/containernetworking/plugins v1.9.0/go.mod h1:JG3BxoJifxxHBhG3hFyxyhid7JgRVBu/wtooGEvWf1c= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,6 +51,8 @@ github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto= +github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -55,6 +62,10 @@ github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6 h1:EEHtgt9IwisQ2AZ4pI github.com/google/pprof v0.0.0-20250820193118-f64d9cf942d6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= @@ -80,6 +91,8 @@ github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/spdystream v0.5.0 h1:7r0J1Si3QO/kjRitvSLVVFUjxMEb/YLj6S9FF62JBCU= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -88,16 +101,21 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= -github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38 h1:UMblx+pkFZ1RA31arHZOWLyWUe6LROdlya4JxTHxGbo= +github.com/openshift-eng/openshift-tests-extension v0.0.0-20260521151256-b5a8f7ec8a38/go.mod h1:pHOS9c6BjZv91OkkHyIHAOWnYhxwcxWQkyYGEvPyUCE= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565 h1:3/q8qM4HbFa+Een8wgzpwO8W6mO7Po+MwY6uxiXi/ac= +github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20260303184444-1cc650aa0565/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -108,14 +126,20 @@ github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzM github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -153,6 +177,8 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= @@ -175,6 +201,8 @@ golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= @@ -192,6 +220,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 000000000..8f2888c28 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,44 @@ +# Makefile for Multus CNI E2E Test + +# Binary name +BINARY_NAME := multus-cni-tests-ext + +# Build directory +BUILD_DIR := bin + +# Go module and package +GO_MODULE := gopkg.in/k8snetworkplumbingwg/multus-cni.v4 + +# Version information +GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') + +# Go build flags +LDFLAGS := -X '$(GO_MODULE)/pkg/version.commitFromGit=$(GIT_COMMIT)' \ + -X '$(GO_MODULE)/pkg/version.buildDate=$(BUILD_DATE)' \ + -X '$(GO_MODULE)/pkg/version.versionFromGit=$(GIT_COMMIT)' + +# Default target +.PHONY: all +all: build + +# Build the binary +.PHONY: build +build: + @echo "Building $(BINARY_NAME)..." + @mkdir -p $(BUILD_DIR) + cd cmd && go build -o ../$(BUILD_DIR)/$(BINARY_NAME) -ldflags="$(LDFLAGS)" . + +# Build for Linux (useful for container builds) +.PHONY: build-linux +build-linux: + @echo "Building $(BINARY_NAME) for Linux..." + @mkdir -p $(BUILD_DIR) + cd cmd && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -o ../$(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 -ldflags="$(LDFLAGS)" . + +# Clean build artifacts +.PHONY: clean +clean: + @echo "Cleaning build artifacts..." + rm -rf $(BUILD_DIR) diff --git a/test/cmd/main.go b/test/cmd/main.go new file mode 100644 index 000000000..ea9650c7b --- /dev/null +++ b/test/cmd/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "os" + + "github.com/openshift-eng/openshift-tests-extension/pkg/cmd" + + "github.com/spf13/cobra" + + e "github.com/openshift-eng/openshift-tests-extension/pkg/extension" + et "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests" + g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo" + + // Import OTP test package + _ "gopkg.in/k8snetworkplumbingwg/multus-cni.v4/test/otp" +) + +func main() { + registry := e.NewRegistry() + + ext := e.NewExtension("openshift", "payload", "multus-cni") + ext.AddSuite(e.Suite{ + Name: "openshift/multus-cni", + Parents: []string{ + "openshift/conformance/parallel", + }, + Qualifiers: []string{ + "name.contains('[Suite:openshift/multus-cni]')", + }, + }) + + specs, err := g.BuildExtensionTestSpecsFromOpenShiftGinkgoSuite() + if err != nil { + fmt.Fprintf(os.Stderr, "couldn't build extension test specs from ginkgo: %v\n", err) + os.Exit(1) + } + + specs.Walk(func(spec *et.ExtensionTestSpec) { + spec.Lifecycle = et.LifecycleInforming + }) + ext.AddSpecs(specs) + registry.Register(ext) + + root := &cobra.Command{ + Long: "OpenShift Tests Extension for Multus CNI", + } + root.AddCommand(cmd.DefaultExtensionCommands(registry)...) + + if err := func() error { + return root.Execute() + }(); err != nil { + os.Exit(1) + } +} diff --git a/test/otp/multus.go b/test/otp/multus.go new file mode 100644 index 000000000..198b9276e --- /dev/null +++ b/test/otp/multus.go @@ -0,0 +1,1540 @@ +package otp + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" +) + +var _ = g.Describe("[sig-network][OTP][Suite:openshift/conformance/parallel] Multus CNI", func() { + var ( + clientset *kubernetes.Clientset + config *rest.Config + ctx context.Context + ) + + g.BeforeEach(func() { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute) + g.DeferCleanup(cancel) + + // Load kubeconfig + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + + var err error + config, err = kubeConfig.ClientConfig() + o.Expect(err).NotTo(o.HaveOccurred()) + + clientset, err = kubernetes.NewForConfig(config) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + // High-57589: Whereabouts CNI Timeout with Large Exclude Range + g.It("[JIRA:Networking][OTP] 57589-should handle large IPv6 exclude ranges without timeout", func() { + const testNS = "test-whereabouts-57589" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NetworkAttachmentDefinition with large exclude range") + nadConfig := `{ + "cniVersion": "0.3.1", + "name": "bridge-net", + "type": "bridge", + "bridge": "test-br0", + "isGateway": false, + "ipMasq": false, + "ipam": { + "type": "whereabouts", + "range": "fd43:01f1:3daa:0baa::/64", + "exclude": [ "fd43:01f1:3daa:0baa::/100" ], + "log_file": "/tmp/whereabouts.log", + "log_level" : "debug" + } + }` + + err = createNAD(ctx, config, testNS, "nad-w-excludes", nadConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating pod with secondary network") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "nad-w-excludes", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pod to reach Running state (max 60s)") + // Pod should be Running within 60 seconds (test validates no timeout) + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), + "Pod did not reach Running state within 60s - Whereabouts may have timed out") + + g.By("Verifying secondary network attachment") + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-pod", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + networkStatus, ok := p.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod missing network-status annotation") + o.Expect(networkStatus).NotTo(o.BeEmpty()) + + // Verify at least 2 networks (primary + secondary) + networkCount := strings.Count(networkStatus, `"name"`) + o.Expect(networkCount).To(o.BeNumerically(">=", 2), + "Expected at least 2 networks, got %d", networkCount) + }) + + // Medium-76652: Dummy CNI Support + g.It("[JIRA:Networking][OTP] 76652-should support Dummy CNI plugin with Multus", func() { + const testNS = "test-dummy-cni-76652" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NetworkAttachmentDefinition with dummy CNI and static IPAM") + dummyConfig := `{ + "cniVersion": "0.3.1", + "name": "dummy-net", + "type": "dummy", + "ipam": { + "type": "static", + "addresses": [ + { + "address": "10.10.10.2/24" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "dummy-net", dummyConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating pod with dummy network attached") + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-dummy-pod", + Namespace: testNS, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "dummy-net", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, pod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for pod to reach Running state") + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-dummy-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), + "Pod did not reach Running state within 60s") + + g.By("Verifying dummy network interface is created") + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, "test-dummy-pod", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + networkStatus, ok := p.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod missing network-status annotation") + o.Expect(networkStatus).NotTo(o.BeEmpty()) + + g.By("Validating dummy interface has correct IP and configuration") + // Network status should contain 2 interfaces: ovn-kubernetes (primary) + dummy-net (secondary) + o.Expect(networkStatus).To(o.ContainSubstring("ovn-kubernetes"), "Should have primary OVN network") + o.Expect(networkStatus).To(o.ContainSubstring("dummy-net"), "Should have dummy network") + o.Expect(networkStatus).To(o.ContainSubstring("10.10.10.2"), "Should have assigned dummy IP") + + // Verify we have at least 2 network interfaces + networkCount := strings.Count(networkStatus, `"name"`) + o.Expect(networkCount).To(o.BeNumerically(">=", 2), + "Expected at least 2 networks (primary + dummy), got %d", networkCount) + }) + + // Medium-66876: Support Dual Stack IP assignment for whereabouts CNI/IPAM + g.It("[JIRA:Networking][OTP] 66876-should assign dual-stack IPs with Whereabouts IPAM", func() { + const testNS = "test-whereabouts-dualstack-66876" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NetworkAttachmentDefinition with dual-stack Whereabouts IPAM") + dualStackConfig := `{ + "cniVersion": "0.3.1", + "name": "whereabouts-dualstack", + "type": "macvlan", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "ipRanges": [ + { + "range": "192.168.10.0/24" + }, + { + "range": "fd00:dead:beef:10::/64" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "whereabouts-dualstack", dualStackConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating deployment with 2 pods using pod affinity for same-node placement") + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-pod", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "test-pod", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "test-pod", + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "test-pod", + Image: "registry.access.redhat.com/ubi9/python-39:latest", + Command: []string{"/bin/bash", "-c"}, + Args: []string{ + `cat > /tmp/server.py <<'PYEOF' +import http.server +import socketserver +import socket +PORT = 8080 +class DualStackTCPServer(socketserver.TCPServer): + address_family = socket.AF_INET6 + def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): + super().__init__(server_address, RequestHandlerClass, bind_and_activate=False) + self.socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + if bind_and_activate: + self.server_bind() + self.server_activate() +class Handler(http.server.SimpleHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'text/plain') + self.end_headers() + self.wfile.write(b'whereabouts-dualstack-test-pod\n') +with DualStackTCPServer(("::", PORT), Handler) as httpd: + httpd.serve_forever() +PYEOF +python3 /tmp/server.py`, + }, + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.AppsV1().Deployments(testNS).Create(ctx, deployment, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for both pods to reach Running state") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=test-pod", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningCount++ + } + } + return runningCount + }, 120, 10).Should(o.Equal(2), "Both pods should reach Running state") + + g.By("Verifying pods have dual-stack IPs on secondary interface") + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "app=test-pod", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have 2 pods") + + ipv4Assigned := 0 + ipv6Assigned := 0 + var podIPs []string + + for _, pod := range pods.Items { + networkStatus, ok := pod.Annotations["k8s.v1.cni.cncf.io/network-status"] + o.Expect(ok).To(o.BeTrue(), "Pod %s should have network-status annotation", pod.Name) + + // Check for dual-stack IPs in network status + hasIPv4 := strings.Contains(networkStatus, "192.168.10.") + hasIPv6 := strings.Contains(networkStatus, "fd00:dead:beef:10::") + + if hasIPv4 { + ipv4Assigned++ + } + if hasIPv6 { + ipv6Assigned++ + } + + o.Expect(hasIPv4 && hasIPv6).To(o.BeTrue(), + "Pod %s should have both IPv4 (192.168.10.x) and IPv6 (fd00:dead:beef:10::x) addresses", pod.Name) + + // Extract IPv4 for uniqueness check + if hasIPv4 { + ipv4Regex := regexp.MustCompile(`192\.168\.10\.\d+`) + matches := ipv4Regex.FindString(networkStatus) + if matches != "" { + podIPs = append(podIPs, matches) + } + } + } + + o.Expect(ipv4Assigned).To(o.Equal(2), "Both pods should have IPv4 addresses") + o.Expect(ipv6Assigned).To(o.Equal(2), "Both pods should have IPv6 addresses") + + g.By("Verifying dual-stack IP uniqueness") + o.Expect(len(podIPs)).To(o.BeNumerically(">=", 2), "Should have extracted at least 2 IPv4 addresses") + + if len(podIPs) >= 2 { + o.Expect(podIPs[0]).NotTo(o.Equal(podIPs[1]), "Pods should have different IPv4 addresses") + } + + g.By("Testing IPv4 connectivity between pods on the same node") + // Both pods are guaranteed to be on the same node via pod affinity + // Macvlan in bridge mode requires same-node for L2 connectivity + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have exactly 2 pods") + + ipv4Regex := regexp.MustCompile(`192\.168\.10\.\d+`) + ipv6Regex := regexp.MustCompile(`fd00:dead:beef:10::[a-f0-9]+`) + + // Extract IPs from pod 0 and pod 1 + srcPod := pods.Items[0].Name + networkStatus1 := pods.Items[1].Annotations["k8s.v1.cni.cncf.io/network-status"] + + dstIPv4 := ipv4Regex.FindString(networkStatus1) + dstIPv6 := ipv6Regex.FindString(networkStatus1) + + o.Expect(dstIPv4).NotTo(o.BeEmpty(), "Pod 1 should have IPv4 address") + o.Expect(dstIPv6).NotTo(o.BeEmpty(), "Pod 1 should have IPv6 address") + + // Verify both pods are on the same node (should always be true due to affinity) + o.Expect(pods.Items[0].Spec.NodeName).To(o.Equal(pods.Items[1].Spec.NodeName), + "Both pods should be on the same node due to pod affinity") + + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Test IPv4 connectivity + curlCmd := []string{"curl", "-s", "--connect-timeout", "5", fmt.Sprintf("http://%s:8080", dstIPv4)} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(srcPod). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "test-pod", + Command: curlCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "IPv4 connectivity test failed: %s", stderr.String()) + o.Expect(stdout.String()).To(o.ContainSubstring("whereabouts-dualstack-test-pod"), + "IPv4 connectivity: Expected response from hello-sdn server") + + g.By("Testing IPv6 connectivity between pods on secondary network") + // Test IPv6 connectivity - curl requires brackets around IPv6 and -g flag + curlCmd = []string{"curl", "-s", "-6", "-g", "--connect-timeout", "5", fmt.Sprintf("http://[%s]:8080", dstIPv6)} + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(srcPod). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "test-pod", + Command: curlCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "IPv6 connectivity test failed: %s", stderr.String()) + o.Expect(stdout.String()).To(o.ContainSubstring("whereabouts-dualstack-test-pod"), + "IPv6 connectivity: Expected response from hello-sdn server") + }) + + // OCP-69947: Macvlan pods send Unsolicited Neighbor Advertisements + // Note: Marked as informing due to timing sensitivity with tcpdump in automated environment + g.It("[JIRA:Networking][OTP] 69947-should send Unsolicited Neighbor Advertisements when macvlan pod is created", func() { + // AWS Limitation: AWS VPC doesn't support L2 IPv6 multicast/NDP required for macvlan NAs. + // This test validates ICMPv6 Neighbor Advertisement packets sent when macvlan pods are created, + // which requires L2 network capabilities that AWS VPC blocks at the hypervisor level. + // Test will SKIP on AWS and should PASS on bare metal, VMware, or other L2-capable platforms. + // To verify this test actually works (not just skips), run on non-AWS infrastructure where + // L2 multicast is supported. Check OTP CI results on bare metal clusters for validation. + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{Limit: 1}) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(nodes.Items) > 0 { + providerID := nodes.Items[0].Spec.ProviderID + if providerID != "" && (strings.Contains(providerID, "aws") || strings.Contains(providerID, "ec2")) { + g.Skip("Skipping on AWS: AWS VPC doesn't support L2 IPv6 multicast/NDP required for macvlan Unsolicited Neighbor Advertisements") + } + } + + testNS := "test-macvlan-na-69947" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err = clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NetworkAttachmentDefinition with dual-stack whereabouts IPAM") + nadConfig := `{ + "cniVersion": "0.3.1", + "name": "whereabouts-dualstack", + "type": "macvlan", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "ipRanges": [ + { + "range": "192.168.10.0/24" + }, + { + "range": "fd00:dead:beef:10::/64" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "whereabouts-dualstack", nadConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating sniffer pod to capture ICMPv6 Neighbor Advertisements") + snifferPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sniff-pod", + Namespace: testNS, + Labels: map[string]string{ + "app": "sniffer", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "sniffer", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Command: []string{"/bin/sh", "-c"}, + Args: []string{ + // Start tcpdump to capture ICMPv6 Neighbor Advertisements on net1 + // Filter: icmp6 type 136 (Neighbor Advertisement) + `tcpdump -i net1 -n 'icmp6 and icmp6[0] = 136' -w /tmp/capture.pcap & + sleep 3600`, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"NET_RAW", "NET_ADMIN"}, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, snifferPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for sniffer pod to be running + o.Eventually(func() corev1.PodPhase { + pod, err := clientset.CoreV1().Pods(testNS).Get(ctx, "sniff-pod", metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return pod.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), "Sniffer pod should be running") + + // Give tcpdump time to start capturing + // tcpdump needs extra time after pod reaches Running to initialize and start listening + time.Sleep(20 * time.Second) + + g.By("Creating 6 test pods with macvlan secondary network") + rc := &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: testNS, + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: int32Ptr(6), + Selector: map[string]string{ + "name": "test-pod", + }, + Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "test-pod", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "whereabouts-dualstack", + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + PodAffinity: &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "sniffer", + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "test-pod", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Env: []corev1.EnvVar{ + { + Name: "RESPONSE", + Value: "Hello", + }, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().ReplicationControllers(testNS).Create(ctx, rc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for all 6 test pods to reach Running state") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=test-pod", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + runningCount++ + } + } + return runningCount + }, 120, 10).Should(o.Equal(6), "All 6 test pods should be running") + + // Wait additional time for Unsolicited Neighbor Advertisements to be sent + time.Sleep(15 * time.Second) + + g.By("Analyzing captured ICMPv6 Neighbor Advertisements") + // Stop tcpdump and read the capture file + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Kill tcpdump process + killCmd := []string{"/bin/sh", "-c", "pkill tcpdump"} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: killCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + if err := exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }); err != nil { + g.GinkgoLogr.Error(err, "Failed to kill tcpdump process", "stdout", stdout.String(), "stderr", stderr.String()) + } + + // Wait for tcpdump to flush pcap file to disk + time.Sleep(5 * time.Second) + + // Read and analyze the pcap file using tcpdump + // Check for ICMPv6 NA packets with solicited flag = 0 (Unsolicited) + analyzeCmd := []string{"/bin/sh", "-c", + `tcpdump -r /tmp/capture.pcap -n 'icmp6 and icmp6[0] = 136' -v 2>/dev/null | grep "Neighbor Advertisement" | wc -l`} + + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: analyzeCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to analyze pcap: %s", stderr.String()) + + naCountStr := strings.TrimSpace(stdout.String()) + naCount, err := strconv.Atoi(naCountStr) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to parse NA count: %s", naCountStr) + o.Expect(naCount).To(o.BeNumerically(">", 0), "Should have captured at least one ICMPv6 Neighbor Advertisement") + + g.By("Verifying Neighbor Advertisements are Unsolicited (solicited flag = 0)") + // Check that captured NAs have solicited flag = 0 + // In unsolicited NA, the destination is ff02::1 (all nodes multicast) + verifyCmd := []string{"/bin/sh", "-c", + `tcpdump -r /tmp/capture.pcap -n 'icmp6 and icmp6[0] = 136' 2>/dev/null | grep "ff02::1" | wc -l`} + + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name("sniff-pod"). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "sniffer", + Command: verifyCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to verify unsolicited NAs: %s", stderr.String()) + + unsolicitedCountStr := strings.TrimSpace(stdout.String()) + unsolicitedCount, err := strconv.Atoi(unsolicitedCountStr) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to parse unsolicited NA count: %s", unsolicitedCountStr) + o.Expect(unsolicitedCount).To(o.BeNumerically(">", 0), + "Should have captured Unsolicited Neighbor Advertisements (destination ff02::1)") + }) + + // OCP-80524: Verify pods with isolated port using bridge-cni + g.It("[JIRA:Networking][OTP] 80524-should isolate pods with portIsolation enabled on bridge CNI", func() { + testNS := "test-bridge-port-isolation-80524" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NetworkAttachmentDefinition with portIsolation enabled") + nadConfig := `{ + "cniVersion": "0.4.0", + "name": "bridge-isolated-ports", + "type": "bridge", + "portIsolation": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "192.168.10.0/24", + "rangeStart": "192.168.10.1", + "rangeEnd": "192.168.10.100" + } + ], + [ + { + "subnet": "FD00:192:168:10::0/64", + "rangeStart": "FD00:192:168:10::1", + "rangeEnd": "FD00:192:168:10::100" + } + ] + ] + } + }` + + err = createNAD(ctx, config, testNS, "bridge-isolated-ports", nadConfig) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating ReplicationController with 2 pods on the same node") + // Get a schedulable node (works on SNO, standard HA, and HyperShift) + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(nodes.Items)).To(o.BeNumerically(">", 0), "Should have at least one node") + + // Find first Ready, schedulable node (no taints blocking scheduling) + var targetNode string + for _, node := range nodes.Items { + // Check if node is Ready + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + // Check if node is schedulable (not cordoned) + if !node.Spec.Unschedulable { + targetNode = node.Name + break + } + } + } + if targetNode != "" { + break + } + } + o.Expect(targetNode).NotTo(o.BeEmpty(), "Should have at least one Ready, schedulable node") + + rc := &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "red-test-pod", + Namespace: testNS, + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: int32Ptr(2), + Selector: map[string]string{ + "name": "red", + }, + Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "red", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "bridge-isolated-ports", + }, + }, + Spec: corev1.PodSpec{ + NodeName: targetNode, + Containers: []corev1.Container{ + { + Name: "red-test-pod", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + {ContainerPort: 443}, + }, + Env: []corev1.EnvVar{ + { + Name: "RESPONSE", + Value: "red-test-pod", + }, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().ReplicationControllers(testNS).Create(ctx, rc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for both pods to be Running") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=red", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Spec.NodeName == targetNode { + runningCount++ + } + } + return runningCount + }, 120, 5).Should(o.Equal(2), "Both pods should be running on the same node") + + g.By("Getting pod IPs from secondary network") + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=red", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have exactly 2 pods") + + pod1 := pods.Items[0] + pod2 := pods.Items[1] + + // Parse network status annotation to get secondary IP + var pod1SecondaryIP, pod2SecondaryIP string + if netStatus, ok := pod1.Annotations["k8s.v1.cni.cncf.io/network-status"]; ok { + var networks []map[string]interface{} + err = json.Unmarshal([]byte(netStatus), &networks) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, net := range networks { + if name, ok := net["name"].(string); ok && name == testNS+"/bridge-isolated-ports" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod1SecondaryIP = ips[0].(string) + } + } + } + } + o.Expect(pod1SecondaryIP).NotTo(o.BeEmpty(), "Pod1 should have secondary IP") + + if netStatus, ok := pod2.Annotations["k8s.v1.cni.cncf.io/network-status"]; ok { + var networks []map[string]interface{} + err = json.Unmarshal([]byte(netStatus), &networks) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, net := range networks { + if name, ok := net["name"].(string); ok && name == testNS+"/bridge-isolated-ports" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod2SecondaryIP = ips[0].(string) + } + } + } + } + o.Expect(pod2SecondaryIP).NotTo(o.BeEmpty(), "Pod2 should have secondary IP") + + g.By("Verifying pods cannot communicate via isolated bridge ports") + // Try to ping pod2 from pod1 using secondary network IP + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + pingCmd := []string{"ping", "-c", "3", "-W", "2", pod2SecondaryIP} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod1.Name). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "red-test-pod", + Command: pingCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + + // Ping should FAIL because ports are isolated + o.Expect(err).To(o.HaveOccurred(), "Ping should fail between isolated ports") + output := stdout.String() + stderr.String() + o.Expect(output).To(o.Or( + o.ContainSubstring("100% packet loss"), + o.ContainSubstring("Network is unreachable"), + ), "Should show network isolation") + }) + + g.It("[JIRA:Networking][OTP] 80525-should allow communication on non-isolated network but not on isolated network", func() { + testNS := "test-bridge-mixed-isolation-80525" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Creating NAD with portIsolation enabled") + nadIsolated := `{ + "cniVersion": "0.4.0", + "name": "bridge-isolated-ports", + "type": "bridge", + "portIsolation": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "192.168.10.0/24", + "rangeStart": "192.168.10.1", + "rangeEnd": "192.168.10.100" + } + ], + [ + { + "subnet": "FD00:192:168:10::0/64", + "rangeStart": "FD00:192:168:10::1", + "rangeEnd": "FD00:192:168:10::100" + } + ] + ] + } + }` + + err = createNAD(ctx, config, testNS, "bridge-isolated-ports", nadIsolated) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating NAD with portIsolation disabled") + nadNonIsolated := `{ + "cniVersion": "0.4.0", + "name": "bridge-whereabouts", + "portIsolation": false, + "type": "bridge", + "ipam": { + "type": "whereabouts", + "ipRanges": [ + { + "range": "192.168.14.0/24", + "rangeStart": "192.168.14.1", + "rangeEnd": "192.168.14.100" + }, + { + "range": "FD00:192:168:14::0/64", + "rangeStart": "FD00:192:168:14::1", + "rangeEnd": "FD00:192:168:14::100" + } + ] + } + }` + + err = createNAD(ctx, config, testNS, "bridge-whereabouts", nadNonIsolated) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Creating ReplicationController with 2 pods using both NADs on the same node") + // Get a schedulable node (works on SNO, standard HA, and HyperShift) + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(nodes.Items)).To(o.BeNumerically(">", 0), "Should have at least one node") + + // Find first Ready, schedulable node + var targetNode string + for _, node := range nodes.Items { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + if !node.Spec.Unschedulable { + targetNode = node.Name + break + } + } + } + if targetNode != "" { + break + } + } + o.Expect(targetNode).NotTo(o.BeEmpty(), "Should have at least one Ready, schedulable node") + + rc := &corev1.ReplicationController{ + ObjectMeta: metav1.ObjectMeta{ + Name: "green-test-pod", + Namespace: testNS, + }, + Spec: corev1.ReplicationControllerSpec{ + Replicas: int32Ptr(2), + Selector: map[string]string{ + "name": "green", + }, + Template: &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "name": "green", + }, + Annotations: map[string]string{ + "k8s.v1.cni.cncf.io/networks": "bridge-isolated-ports, bridge-whereabouts", + }, + }, + Spec: corev1.PodSpec{ + NodeName: targetNode, + Containers: []corev1.Container{ + { + Name: "green-test-pod", + Image: "quay.io/openshifttest/hello-sdn@sha256:c89445416459e7adea9a5a416b3365ed3d74f2491beb904d61dc8d1eb89a72a4", + Ports: []corev1.ContainerPort{ + {ContainerPort: 8080}, + {ContainerPort: 443}, + }, + Env: []corev1.EnvVar{ + { + Name: "RESPONSE", + Value: "green-test-pod", + }, + }, + }, + }, + }, + }, + }, + } + + _, err = clientset.CoreV1().ReplicationControllers(testNS).Create(ctx, rc, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Waiting for both pods to be Running") + o.Eventually(func() int { + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=green", + }) + if err != nil { + return 0 + } + runningCount := 0 + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning && pod.Spec.NodeName == targetNode { + runningCount++ + } + } + return runningCount + }, 120, 5).Should(o.Equal(2), "Both pods should be running on the same node") + + g.By("Getting pod IPs from both networks") + pods, err := clientset.CoreV1().Pods(testNS).List(ctx, metav1.ListOptions{ + LabelSelector: "name=green", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(pods.Items)).To(o.Equal(2), "Should have exactly 2 pods") + + pod1 := pods.Items[0] + pod2 := pods.Items[1] + + // Parse network status to get IPs from both networks + var pod1IsolatedIP, pod1NonIsolatedIP, pod2IsolatedIP, pod2NonIsolatedIP string + + if netStatus, ok := pod1.Annotations["k8s.v1.cni.cncf.io/network-status"]; ok { + var networks []map[string]interface{} + err = json.Unmarshal([]byte(netStatus), &networks) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, net := range networks { + if name, ok := net["name"].(string); ok { + if name == testNS+"/bridge-isolated-ports" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod1IsolatedIP = ips[0].(string) + } + } else if name == testNS+"/bridge-whereabouts" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod1NonIsolatedIP = ips[0].(string) + } + } + } + } + } + o.Expect(pod1IsolatedIP).NotTo(o.BeEmpty(), "Pod1 should have isolated network IP") + o.Expect(pod1NonIsolatedIP).NotTo(o.BeEmpty(), "Pod1 should have non-isolated network IP") + + if netStatus, ok := pod2.Annotations["k8s.v1.cni.cncf.io/network-status"]; ok { + var networks []map[string]interface{} + err = json.Unmarshal([]byte(netStatus), &networks) + o.Expect(err).NotTo(o.HaveOccurred()) + for _, net := range networks { + if name, ok := net["name"].(string); ok { + if name == testNS+"/bridge-isolated-ports" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod2IsolatedIP = ips[0].(string) + } + } else if name == testNS+"/bridge-whereabouts" { + if ips, ok := net["ips"].([]interface{}); ok && len(ips) > 0 { + pod2NonIsolatedIP = ips[0].(string) + } + } + } + } + } + o.Expect(pod2IsolatedIP).NotTo(o.BeEmpty(), "Pod2 should have isolated network IP") + o.Expect(pod2NonIsolatedIP).NotTo(o.BeEmpty(), "Pod2 should have non-isolated network IP") + + scheme := runtime.NewScheme() + err = corev1.AddToScheme(scheme) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Verifying pods CANNOT communicate via isolated network") + pingIsolatedCmd := []string{"ping", "-c", "3", "-W", "2", pod2IsolatedIP} + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod1.Name). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "green-test-pod", + Command: pingIsolatedCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + + // Ping should FAIL on isolated network + o.Expect(err).To(o.HaveOccurred(), "Ping should fail on isolated network") + isolatedOutput := stdout.String() + stderr.String() + o.Expect(isolatedOutput).To(o.Or( + o.ContainSubstring("100% packet loss"), + o.ContainSubstring("Network is unreachable"), + ), "Should show network isolation on isolated network") + + g.By("Verifying pods CAN communicate via non-isolated network") + pingNonIsolatedCmd := []string{"ping", "-c", "3", "-W", "2", pod2NonIsolatedIP} + req = clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod1.Name). + Namespace(testNS). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: "green-test-pod", + Command: pingNonIsolatedCmd, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err = remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + o.Expect(err).NotTo(o.HaveOccurred()) + + stdout.Reset() + stderr.Reset() + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + + // Ping should SUCCEED on non-isolated network + o.Expect(err).NotTo(o.HaveOccurred(), "Ping should succeed on non-isolated network") + nonIsolatedOutput := stdout.String() + stderr.String() + o.Expect(nonIsolatedOutput).To(o.ContainSubstring("0% packet loss"), "Should show successful ping on non-isolated network") + }) + + g.It("[JIRA:Networking][OTP] 77102-should have secure permissions on CNI configuration files", func() { + testNS := "test-cni-permissions-77102" + + g.By("Creating test namespace") + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNS, + Labels: map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", + "security.openshift.io/scc.podSecurityLabelSync": "false", + }, + }, + } + _, err := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + defer func() { + g.By("Cleaning up test namespace") + if err := clientset.CoreV1().Namespaces().Delete(ctx, testNS, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + g.GinkgoLogr.Error(err, "Failed to delete test namespace", "namespace", testNS) + } + }() + + g.By("Checking multus config permissions via multus pods") + multusPods, err := clientset.CoreV1().Pods("openshift-multus").List(ctx, metav1.ListOptions{ + LabelSelector: "app=multus", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(multusPods.Items)).To(o.BeNumerically(">", 0), "Expected at least one multus pod") + + // Check first multus pod for config file permissions + multusPod := multusPods.Items[0].Name + output, err := execInPod(ctx, clientset, config, "openshift-multus", multusPod, "kube-multus", + []string{"/bin/bash", "-c", "stat -c '%a %n' /host/etc/cni/net.d/*.conf"}) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to check multus config permissions") + + g.By("Verifying multus config has 600 permissions") + lines := strings.Split(strings.TrimSpace(output), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Fields(line) + o.Expect(len(parts)).To(o.BeNumerically(">=", 2), "Invalid stat output: %s", line) + perms := parts[0] + filename := parts[1] + o.Expect(perms).To(o.Equal("600"), + "CIS violation: %s has insecure permissions %s (expected 600)", filename, perms) + } + + g.By("Checking whereabouts config permissions") + // Get a schedulable node (SNO compatible) + nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(nodes.Items)).To(o.BeNumerically(">", 0), "Should have at least one node") + + // Find first Ready, schedulable node + var nodeName string + for _, node := range nodes.Items { + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + if !node.Spec.Unschedulable { + nodeName = node.Name + break + } + } + } + if nodeName != "" { + break + } + } + o.Expect(nodeName).NotTo(o.BeEmpty(), "Should have at least one Ready, schedulable node") + + // Create debug pod on node + debugPodName := "cis-perms-check-77102" + debugPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: debugPodName, + Namespace: testNS, + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + HostNetwork: true, + HostPID: true, + Containers: []corev1.Container{ + { + Name: "debug", + Image: "registry.access.redhat.com/ubi8/ubi-minimal:latest", + Command: []string{"sleep", "300"}, + SecurityContext: &corev1.SecurityContext{ + Privileged: boolPtr(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host", + MountPath: "/host", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "host", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } + + _, err = clientset.CoreV1().Pods(testNS).Create(ctx, debugPod, metav1.CreateOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Wait for debug pod to be running + o.Eventually(func() corev1.PodPhase { + p, err := clientset.CoreV1().Pods(testNS).Get(ctx, debugPodName, metav1.GetOptions{}) + if err != nil { + return corev1.PodPending + } + return p.Status.Phase + }, 60, 5).Should(o.Equal(corev1.PodRunning), "Debug pod did not reach Running state") + + // Check whereabouts config file permissions + output, err = execInPod(ctx, clientset, config, testNS, debugPodName, "debug", + []string{"/bin/bash", "-c", "stat -c '%a %n' /host/etc/kubernetes/cni/net.d/whereabouts.d/*.conf /host/etc/kubernetes/cni/net.d/whereabouts.d/*.kubeconfig 2>/dev/null || true"}) + o.Expect(err).NotTo(o.HaveOccurred(), "Failed to check whereabouts config permissions") + + g.By("Verifying whereabouts configs have 600 permissions") + if strings.TrimSpace(output) != "" { + lines = strings.Split(strings.TrimSpace(output), "\n") + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Fields(line) + o.Expect(len(parts)).To(o.BeNumerically(">=", 2), "Invalid stat output: %s", line) + perms := parts[0] + filename := parts[1] + o.Expect(perms).To(o.Equal("600"), + "CIS violation: %s has insecure permissions %s (expected 600)", filename, perms) + } + } + }) +}) + +// createNAD creates a NetworkAttachmentDefinition +func createNAD(ctx context.Context, config *rest.Config, namespace, name, nadConfig string) error { + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + return err + } + + nadGVR := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + + nad := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "config": nadConfig, + }, + }, + } + + _, err = dynamicClient.Resource(nadGVR).Namespace(namespace).Create(ctx, nad, metav1.CreateOptions{}) + return err +} + +// Helper functions + +// int32Ptr returns a pointer to an int32 +func int32Ptr(i int32) *int32 { + return &i +} + +// boolPtr returns a pointer to a bool +func boolPtr(b bool) *bool { + return &b +} + +// execInPod executes a command in a pod and returns the output +func execInPod(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, + namespace, podName, containerName string, command []string) (string, error) { + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + return "", err + } + + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: command, + Stdout: true, + Stderr: true, + }, runtime.NewParameterCodec(scheme)) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return "", err + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return stdout.String() + "\n" + stderr.String(), err + } + + return stdout.String(), nil +}