From 068806d87917f9edbcb3150e7a99a7b6fec94ac3 Mon Sep 17 00:00:00 2001 From: i759715 Date: Thu, 17 Jul 2025 17:00:21 +0300 Subject: [PATCH 1/8] Add tests for IPv6 security groups --- assets/proxy-ipv6/Procfile | 1 + assets/proxy-ipv6/go.mod | 5 + assets/proxy-ipv6/internal-route-manifest.yml | 11 + assets/proxy-ipv6/main.go | 115 ++++++++ assets/proxy-ipv6/manifest.yml | 10 + cats_suite_helpers/cats_suite_helpers.go | 11 + helpers/assets/assets.go | 2 + security_groups/ipv6_security_groups.go | 266 ++++++++++++++++++ 8 files changed, 421 insertions(+) create mode 100644 assets/proxy-ipv6/Procfile create mode 100644 assets/proxy-ipv6/go.mod create mode 100644 assets/proxy-ipv6/internal-route-manifest.yml create mode 100644 assets/proxy-ipv6/main.go create mode 100644 assets/proxy-ipv6/manifest.yml create mode 100644 security_groups/ipv6_security_groups.go diff --git a/assets/proxy-ipv6/Procfile b/assets/proxy-ipv6/Procfile new file mode 100644 index 000000000..618ce863a --- /dev/null +++ b/assets/proxy-ipv6/Procfile @@ -0,0 +1 @@ +web: proxy diff --git a/assets/proxy-ipv6/go.mod b/assets/proxy-ipv6/go.mod new file mode 100644 index 000000000..3d0648ee5 --- /dev/null +++ b/assets/proxy-ipv6/go.mod @@ -0,0 +1,5 @@ +module example-apps/proxy + +go 1.23 + +toolchain go1.23.7 diff --git a/assets/proxy-ipv6/internal-route-manifest.yml b/assets/proxy-ipv6/internal-route-manifest.yml new file mode 100644 index 000000000..0b3a9b937 --- /dev/null +++ b/assets/proxy-ipv6/internal-route-manifest.yml @@ -0,0 +1,11 @@ +--- +applications: + - name: proxy + memory: 32M + disk_quota: 32M + buildpack: go_buildpack + env: + GOPACKAGENAME: example-apps/proxy + GOVERSION: go1.23 + routes: + - route: app-smoke.apps.internal diff --git a/assets/proxy-ipv6/main.go b/assets/proxy-ipv6/main.go new file mode 100644 index 000000000..e922676a2 --- /dev/null +++ b/assets/proxy-ipv6/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" +) + +func main() { + systemPortString := os.Getenv("PORT") + systemPort, err := strconv.Atoi(systemPortString) + if err != nil { + log.Fatal("Invalid required env var PORT") + } + + mux := http.NewServeMux() + mux.HandleFunc("/proxy/", ipv6ProxyHandler) + mux.HandleFunc("/https_proxy/", ipv6HttpsProxyHandler) + mux.HandleFunc("/", ipv6InfoHandler(systemPort)) + + server := &http.Server{ + Addr: fmt.Sprintf("[::]:%d", systemPort), // Listen on IPv6 interfaces + Handler: mux, + } + _ = server.ListenAndServe() +} + +func ipv6InfoHandler(port int) http.HandlerFunc { + return func(resp http.ResponseWriter, req *http.Request) { + addrs, err := net.InterfaceAddrs() + if err != nil { + panic(err) + } + addressStrings := []string{} + for _, addr := range addrs { + ip, _, _ := net.ParseCIDR(addr.String()) + if ip.To4() == nil { // Ensure it's not IPv4 + addressStrings = append(addressStrings, ip.String()) + } + } + + respBytes, err := json.Marshal(struct { + ListenAddresses []string + Port int + }{ + ListenAddresses: addressStrings, + Port: port, + }) + if err != nil { + panic(err) + } + _, _ = resp.Write(respBytes) + } +} + +func ipv6ProxyHandler(resp http.ResponseWriter, req *http.Request) { + destination := strings.TrimPrefix(req.URL.Path, "/proxy/") + destination = formatIPv6Destination("http", destination) + handleRequest(destination, resp, req) +} + +func ipv6HttpsProxyHandler(resp http.ResponseWriter, req *http.Request) { + destination := strings.TrimPrefix(req.URL.Path, "/https_proxy/") + destination = formatIPv6Destination("https", destination) + handleRequest(destination, resp, req) +} + +func formatIPv6Destination(proto, destination string) string { + if strings.Contains(destination, ":") { + destination = fmt.Sprintf("[%s]", destination) // Encapsulate IPv6 addresses + } + return fmt.Sprintf("%s://%s", proto, destination) +} + +func handleRequest(destination string, resp http.ResponseWriter, req *http.Request) { + getResp, err := httpClient.Get(destination) + if err != nil { + fmt.Fprintf(os.Stderr, "request failed: %s\n", err) + resp.WriteHeader(http.StatusInternalServerError) + _, _ = resp.Write([]byte(fmt.Sprintf("request failed: %s", err))) + return + } + defer getResp.Body.Close() + + readBytes, err := io.ReadAll(getResp.Body) + if err != nil { + resp.WriteHeader(http.StatusInternalServerError) + _, _ = resp.Write([]byte(fmt.Sprintf("read body failed: %s", err))) + return + } + + _, _ = resp.Write(readBytes) +} + +var httpClient = &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 0, + }).DialContext, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, +} + diff --git a/assets/proxy-ipv6/manifest.yml b/assets/proxy-ipv6/manifest.yml new file mode 100644 index 000000000..4b19212cb --- /dev/null +++ b/assets/proxy-ipv6/manifest.yml @@ -0,0 +1,10 @@ +--- +applications: + - name: proxy + memory: 32M + disk_quota: 32M + buildpacks: + - go_buildpack + env: + GOPACKAGENAME: example-apps/proxy + GOVERSION: go1.23 diff --git a/cats_suite_helpers/cats_suite_helpers.go b/cats_suite_helpers/cats_suite_helpers.go index dee5bd993..333495e0e 100644 --- a/cats_suite_helpers/cats_suite_helpers.go +++ b/cats_suite_helpers/cats_suite_helpers.go @@ -438,6 +438,17 @@ func VolumeServicesDescribe(description string, callback func()) bool { }) } +func IPv6SecurityGroupsDescribe(description string, callback func()) bool { + return Describe("[ipv6 security groups]", func() { + BeforeEach(func() { + if !Config.GetIncludeIPv6() { + Skip(skip_messages.SkipIPv6) + } + }) + Describe(description, callback) + }) +} + func GetNServerResponses(n int, domainName, externalPort1 string) ([]string, error) { var responses []string diff --git a/helpers/assets/assets.go b/helpers/assets/assets.go index 735dc2847..b7728982d 100644 --- a/helpers/assets/assets.go +++ b/helpers/assets/assets.go @@ -32,6 +32,7 @@ type Assets struct { Pora string Php string Proxy string + ProxyIpv6 string R string RubySimple string SecurityGroupBuildpack string @@ -81,6 +82,7 @@ func NewAssets() Assets { Pora: "assets/pora", Php: "assets/php", Proxy: "assets/proxy", + ProxyIpv6: "assets/proxy-ipv6", Python: "assets/python", PythonCrashApp: "assets/python-crash-app", R: "assets/r", diff --git a/security_groups/ipv6_security_groups.go b/security_groups/ipv6_security_groups.go new file mode 100644 index 000000000..3bab22efa --- /dev/null +++ b/security_groups/ipv6_security_groups.go @@ -0,0 +1,266 @@ +package security_groups_test + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net/http" + "os" + "time" + + . "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gbytes" + . "github.com/onsi/gomega/gexec" + + "github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/assets" + "github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name" + "github.com/cloudfoundry/cf-test-helpers/v2/cf" + "github.com/cloudfoundry/cf-test-helpers/v2/workflowhelpers" + + "github.com/cloudfoundry/cf-acceptance-tests/helpers/skip_messages" + +) + +var _ = IPv6SecurityGroupsDescribe("Security Group Tests", func() { + var ( + orgName string + spaceName string + securityGroupName string + serverAppName string + privateHost string + privatePort int + clientAppName string + ) + + BeforeEach(func() { + orgName = TestSetup.RegularUserContext().Org + spaceName = TestSetup.RegularUserContext().Space + }) + + Describe("IPv6 Security Group for Internal Cloud Controller Access", func() { + var appName string + + BeforeEach(func() { + appName = random_name.CATSRandomName("APP") + + By("pushing a proxy app") + Expect(cf.Cf( + "push", appName, + "-b", Config.GetGoBuildpackName(), + "-p", assets.NewAssets().ProxyIpv6, + "-f", assets.NewAssets().ProxyIpv6+"/manifest.yml", + ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + }) + + AfterEach(func() { + Expect(cf.Cf("delete", appName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + deleteSecurityGroup(securityGroupName) + }) + + It("manages whether apps can reach certain IPv6 addresses per ASG configuration", func() { + proxyRequestURL := fmt.Sprintf("%s%s.%s/https_proxy/cloud-controller-ng.service.cf.internal:9024/", Config.Protocol(), appName, Config.GetAppsDomain()) + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: Config.GetSkipSSLValidation(), + }, + }, + } + + By("checking that our app can't initially reach cloud controller over internal address") + assertAppCannotConnect(client, proxyRequestURL) + + By("binding a new IPv6 security group") + securityGroupName = bindCCSecurityGroupIpv6(orgName, spaceName) + + if Config.GetDynamicASGsEnabled() { + By("checking that our app can eventually reach cloud controller over internal address") + assertEventuallyAppCanConnect(client, proxyRequestURL) + } else { + By("because dynamic ASGs are not enabled, validating an app restart is required") + assertAppRestartRequiredForConnect(client, proxyRequestURL, appName) + } + + By("unbinding the security group") + unbindSecurityGroup(securityGroupName, orgName, spaceName) + + if Config.GetDynamicASGsEnabled() { + By("checking that our app eventually cannot reach cloud controller over internal address") + assertEventuallyAppCannotConnect(client, proxyRequestURL) + } else { + By("because dynamic ASGs are not enabled, validating an app restart is required") + assertAppRestartRequiredForDisconnect(client, proxyRequestURL, appName) + } + }) + }) + + Describe("Using container-networking and running security-groups with IPv6", func() { + BeforeEach(func() { + if !Config.GetIncludeContainerNetworking() { + Skip(skip_messages.SkipContainerNetworkingMessage) + } + + serverAppName, privateHost, privatePort = pushServerApp() + clientAppName = pushClientApp() + assertIPv6NetworkingPreconditions(clientAppName, privateHost, privatePort) + }) + + AfterEach(func() { + app_helpers.AppReport(serverAppName) + Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + app_helpers.AppReport(clientAppName) + Expect(cf.Cf("delete", clientAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + deleteSecurityGroup(securityGroupName) + }) + + It("correctly configures ASGs and C2C policy independent of each other in terms of IPv6 env", func() { + By("creating a wide-open ASG") + dest := Destination{ + IP: "4000::/3", // Some random IP that isn't covered by an existing Security Group rule + Protocol: "all", + } + securityGroupName = createSecurityGroup(dest) + privateAddress := Config.GetUnallocatedIPForSecurityGroup() + + By("binding new security group") + bindSecurityGroup(securityGroupName, orgName, spaceName) + + Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + By("Testing that client app cannot connect to the server app using the overlay") + containerIp, containerPort := getAppContainerIpAndPort(serverAppName) + catnipCurlResponse := testAppConnectivity(clientAppName, containerIp, containerPort) + Expect(catnipCurlResponse.ReturnCode).NotTo(Equal(0), "No policy configured but client app can talk to server app using overlay") + + By("Testing that external connectivity to a private IP is not refused (but may be unreachable for other reasons)") + Eventually(func() string { + resp := testAppConnectivity(clientAppName, privateAddress, 80) + return resp.Stderr + }, 3*time.Minute).Should(MatchRegexp("Connection timed out after|No route to host"), "Wide-open ASG configured but app is still refused by private IP") + + By("adding policy") + workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { + Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait()).To(Exit(0)) + Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).ToNot(ContainSubstring(serverAppName)) + Expect(cf.Cf("add-network-policy", clientAppName, serverAppName, "--port", fmt.Sprintf("%d", containerPort), "--protocol", "tcp").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + Expect(string(cf.Cf("netwozrk-policies").Wait().Out.Contents())).To(ContainSubstring(serverAppName)) + }) + + By("Testing that client app can connect to server app using the overlay") + Eventually(func() int { + catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) + return catnipCurlResponse.ReturnCode + }, "5s").Should(Equal(0), "Policy is configured + wide-open ASG but client app cannot talk to server app using overlay") + + By("unbinding the wide-open security group") + unbindSecurityGroup(securityGroupName, orgName, spaceName) + Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + By("Testing that client app can still connect to server app using the overlay") + Eventually(func() int { + catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) + return catnipCurlResponse.ReturnCode + }, 3*time.Minute).Should(Equal(0), "Policy is configured, ASGs are not but client app cannot talk to server app using overlay") + + By("Testing that external connectivity to a private IP is refused") + Eventually(func() string { + resp := testAppConnectivity(clientAppName, privateAddress, 80) + return resp.Stderr + }, 3*time.Minute).Should(MatchRegexp("refused|No route to host| Connection timed out")) + + By("deleting policy") + workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { + Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait()).To(Exit(0)) + Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).To(ContainSubstring(serverAppName)) + Expect(cf.Cf("remove-network-policy", clientAppName, serverAppName, "--port", fmt.Sprintf("%d", containerPort), "--protocol", "tcp").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).ToNot(ContainSubstring(serverAppName)) + }) + + By("Testing the client app cannot connect to the server app using the overlay") + Eventually(func() int { + catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) + return catnipCurlResponse.ReturnCode + }, 3*time.Minute).ShouldNot(Equal(0), "No policy is configured but client app can talk to server app using overlay") + }) + }) + + Describe("Using staging security groups", func() { + var testAppName, buildpack string + + BeforeEach(func() { + serverAppName, privateHost, privatePort = pushServerApp() + + By("Asserting default staging security group configuration") + testAppName = random_name.CATSRandomName("APP") + buildpack = createDummyBuildpack() + pushApp(testAppName, buildpack) + + privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort) + Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait()).To(Exit(0)) + + Expect(cf.Cf("start", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1)) + Eventually(getStagingOutput(testAppName), 5).Should(Say("CURL_EXIT=[^0]"), "Expected staging security groups not to allow internal communication between app containers. Configure your staging security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") + }) + + AfterEach(func() { + app_helpers.AppReport(serverAppName) + Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + app_helpers.AppReport(testAppName) + Expect(cf.Cf("delete", testAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) + + deleteBuildpack(buildpack) + }) + + It("allows external and denies internal traffic during staging based on default IPv6 staging security rules", func() { + Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.ipv6.google.com").Wait()).To(Exit(0)) + Expect(cf.Cf("start", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1)) + Eventually(getStagingOutput(testAppName), 5).Should(Say("CURL_EXIT=0")) + }) + }) +}) + +func bindCCSecurityGroupIpv6(orgName, spaceName string) string { + destinations := []Destination{{ + IP: "2001:0db8::/32", // Adjust as necessary for the environment + Ports: "9024", // Adjust as needed for security context + Protocol: "tcp", + }} + securityGroupName := createSecurityGroupIPv6(destinations...) + bindSecurityGroup(securityGroupName, orgName, spaceName) + + return securityGroupName +} + +func createSecurityGroupIPv6(allowedDestinations ...Destination) string { + file, err := os.CreateTemp(os.TempDir(), "CATS-sg-rules") + Expect(err).NotTo(HaveOccurred()) + defer os.Remove(file.Name()) + + Expect(json.NewEncoder(file).Encode(allowedDestinations)).To(Succeed()) + + rulesPath := file.Name() + securityGroupName := random_name.CATSRandomName("SG") + + workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { + Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait()).To(Exit(0)) + }) + + return securityGroupName +} + +func assertIPv6NetworkingPreconditions(clientAppName string, privateHost string, privatePort int) { + By("Asserting default running security group configuration for traffic between containers") + catnipCurlResponse := testAppConnectivity(clientAppName, privateHost, privatePort) + Expect(catnipCurlResponse.ReturnCode).NotTo(Equal(0), "Expected default running security groups not to allow internal communication between app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") + + By("Asserting default running security group configuration from a running container to an external destination") + catnipCurlResponse = testAppConnectivity(clientAppName, "www.ipv6.google.com", 80) + Expect(catnipCurlResponse.ReturnCode).To(Equal(0), "Expected default running security groups to allow external traffic from app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") +} \ No newline at end of file From 436fa635c0522b030488c7a1fcdcb0b246ce161b Mon Sep 17 00:00:00 2001 From: i759715 Date: Tue, 22 Jul 2025 09:55:52 +0300 Subject: [PATCH 2/8] Modifies test completely only to validate IPv6 egress --- assets/proxy-ipv6/Procfile | 1 - security_groups/ipv6_security_groups.go | 279 +++++------------------- 2 files changed, 53 insertions(+), 227 deletions(-) delete mode 100644 assets/proxy-ipv6/Procfile diff --git a/assets/proxy-ipv6/Procfile b/assets/proxy-ipv6/Procfile deleted file mode 100644 index 618ce863a..000000000 --- a/assets/proxy-ipv6/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: proxy diff --git a/security_groups/ipv6_security_groups.go b/security_groups/ipv6_security_groups.go index 3bab22efa..f640149cd 100644 --- a/security_groups/ipv6_security_groups.go +++ b/security_groups/ipv6_security_groups.go @@ -1,266 +1,93 @@ package security_groups_test import ( - "crypto/tls" - "encoding/json" - "fmt" - "net/http" - "os" - "time" + "net" + "strings" . "github.com/cloudfoundry/cf-acceptance-tests/cats_suite_helpers" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gbytes" . "github.com/onsi/gomega/gexec" "github.com/cloudfoundry/cf-acceptance-tests/helpers/app_helpers" "github.com/cloudfoundry/cf-acceptance-tests/helpers/assets" "github.com/cloudfoundry/cf-acceptance-tests/helpers/random_name" "github.com/cloudfoundry/cf-test-helpers/v2/cf" - "github.com/cloudfoundry/cf-test-helpers/v2/workflowhelpers" - - "github.com/cloudfoundry/cf-acceptance-tests/helpers/skip_messages" - + "github.com/cloudfoundry/cf-test-helpers/v2/helpers" ) -var _ = IPv6SecurityGroupsDescribe("Security Group Tests", func() { +var _ = IPv6SecurityGroupsDescribe("IPv6 Security Group", func() { var ( + appName string + securityGroupName string orgName string spaceName string - securityGroupName string - serverAppName string - privateHost string - privatePort int - clientAppName string ) BeforeEach(func() { + appName = random_name.CATSRandomName("APP-IPv6") orgName = TestSetup.RegularUserContext().Org spaceName = TestSetup.RegularUserContext().Space + securityGroupName = "ipv6_public_networks" + + By("pushing simple python app") + Expect(cf.Cf( + "push", appName, + "-p", assets.NewAssets().Python, + "-m", DEFAULT_MEMORY_LIMIT, + ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) }) - Describe("IPv6 Security Group for Internal Cloud Controller Access", func() { - var appName string - - BeforeEach(func() { - appName = random_name.CATSRandomName("APP") - - By("pushing a proxy app") - Expect(cf.Cf( - "push", appName, - "-b", Config.GetGoBuildpackName(), - "-p", assets.NewAssets().ProxyIpv6, - "-f", assets.NewAssets().ProxyIpv6+"/manifest.yml", - ).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - }) - - AfterEach(func() { - Expect(cf.Cf("delete", appName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - deleteSecurityGroup(securityGroupName) - }) - - It("manages whether apps can reach certain IPv6 addresses per ASG configuration", func() { - proxyRequestURL := fmt.Sprintf("%s%s.%s/https_proxy/cloud-controller-ng.service.cf.internal:9024/", Config.Protocol(), appName, Config.GetAppsDomain()) - - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: Config.GetSkipSSLValidation(), - }, - }, - } - - By("checking that our app can't initially reach cloud controller over internal address") - assertAppCannotConnect(client, proxyRequestURL) - - By("binding a new IPv6 security group") - securityGroupName = bindCCSecurityGroupIpv6(orgName, spaceName) - - if Config.GetDynamicASGsEnabled() { - By("checking that our app can eventually reach cloud controller over internal address") - assertEventuallyAppCanConnect(client, proxyRequestURL) - } else { - By("because dynamic ASGs are not enabled, validating an app restart is required") - assertAppRestartRequiredForConnect(client, proxyRequestURL, appName) - } - - By("unbinding the security group") - unbindSecurityGroup(securityGroupName, orgName, spaceName) - - if Config.GetDynamicASGsEnabled() { - By("checking that our app eventually cannot reach cloud controller over internal address") - assertEventuallyAppCannotConnect(client, proxyRequestURL) - } else { - By("because dynamic ASGs are not enabled, validating an app restart is required") - assertAppRestartRequiredForDisconnect(client, proxyRequestURL, appName) - } - }) + AfterEach(func() { + app_helpers.AppReport(appName) + Expect(cf.Cf("delete", appName, "-f", "-r").Wait()).To(Exit(0)) }) - Describe("Using container-networking and running security-groups with IPv6", func() { - BeforeEach(func() { - if !Config.GetIncludeContainerNetworking() { - Skip(skip_messages.SkipContainerNetworkingMessage) - } - - serverAppName, privateHost, privatePort = pushServerApp() - clientAppName = pushClientApp() - assertIPv6NetworkingPreconditions(clientAppName, privateHost, privatePort) + assertAppCanConnect := func() { + response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv4-test") + responseParts := strings.Split(response, "\n") + ipAddress := responseParts[0] + statusCode := responseParts[1] + + parsedIP := net.ParseIP(ipAddress) + Expect(parsedIP).NotTo(BeNil(), "Expected a valid IP address") + Expect(statusCode).To(Equal("200")) + } + + assertAppCanNotConnect := func() { + response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv4-test") + responseParts := strings.Split(response, "\n") + ipAddress := responseParts[0] + statusCode := responseParts[1] + + bodyResponce := net.ParseIP(ipAddress) + Expect(bodyResponce).To(BeNil(), "Expected a non-valid IP address") + Expect(bodyResponce).To(ContainSubstring("not supported by protocol")) + Expect(statusCode).To(Equal("500")) + } + + Describe("Default IPv6 security groups are working", func() { + It("validates IPv6 with security groups enabled", func() { + assertAppCanConnect() }) - AfterEach(func() { - app_helpers.AppReport(serverAppName) - Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - app_helpers.AppReport(clientAppName) - Expect(cf.Cf("delete", clientAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - deleteSecurityGroup(securityGroupName) - }) - - It("correctly configures ASGs and C2C policy independent of each other in terms of IPv6 env", func() { - By("creating a wide-open ASG") - dest := Destination{ - IP: "4000::/3", // Some random IP that isn't covered by an existing Security Group rule - Protocol: "all", - } - securityGroupName = createSecurityGroup(dest) - privateAddress := Config.GetUnallocatedIPForSecurityGroup() - - By("binding new security group") - bindSecurityGroup(securityGroupName, orgName, spaceName) - - Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - By("Testing that client app cannot connect to the server app using the overlay") - containerIp, containerPort := getAppContainerIpAndPort(serverAppName) - catnipCurlResponse := testAppConnectivity(clientAppName, containerIp, containerPort) - Expect(catnipCurlResponse.ReturnCode).NotTo(Equal(0), "No policy configured but client app can talk to server app using overlay") - - By("Testing that external connectivity to a private IP is not refused (but may be unreachable for other reasons)") - Eventually(func() string { - resp := testAppConnectivity(clientAppName, privateAddress, 80) - return resp.Stderr - }, 3*time.Minute).Should(MatchRegexp("Connection timed out after|No route to host"), "Wide-open ASG configured but app is still refused by private IP") - - By("adding policy") - workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { - Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait()).To(Exit(0)) - Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).ToNot(ContainSubstring(serverAppName)) - Expect(cf.Cf("add-network-policy", clientAppName, serverAppName, "--port", fmt.Sprintf("%d", containerPort), "--protocol", "tcp").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - Expect(string(cf.Cf("netwozrk-policies").Wait().Out.Contents())).To(ContainSubstring(serverAppName)) - }) - - By("Testing that client app can connect to server app using the overlay") - Eventually(func() int { - catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) - return catnipCurlResponse.ReturnCode - }, "5s").Should(Equal(0), "Policy is configured + wide-open ASG but client app cannot talk to server app using overlay") - + It("unbinds the wide-open security group", func() { By("unbinding the wide-open security group") unbindSecurityGroup(securityGroupName, orgName, spaceName) - Expect(cf.Cf("restart", clientAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - By("Testing that client app can still connect to server app using the overlay") - Eventually(func() int { - catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) - return catnipCurlResponse.ReturnCode - }, 3*time.Minute).Should(Equal(0), "Policy is configured, ASGs are not but client app cannot talk to server app using overlay") - - By("Testing that external connectivity to a private IP is refused") - Eventually(func() string { - resp := testAppConnectivity(clientAppName, privateAddress, 80) - return resp.Stderr - }, 3*time.Minute).Should(MatchRegexp("refused|No route to host| Connection timed out")) - - By("deleting policy") - workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { - Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait()).To(Exit(0)) - Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).To(ContainSubstring(serverAppName)) - Expect(cf.Cf("remove-network-policy", clientAppName, serverAppName, "--port", fmt.Sprintf("%d", containerPort), "--protocol", "tcp").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - Expect(string(cf.Cf("network-policies").Wait().Out.Contents())).ToNot(ContainSubstring(serverAppName)) - }) - - By("Testing the client app cannot connect to the server app using the overlay") - Eventually(func() int { - catnipCurlResponse = testAppConnectivity(clientAppName, containerIp, containerPort) - return catnipCurlResponse.ReturnCode - }, 3*time.Minute).ShouldNot(Equal(0), "No policy is configured but client app can talk to server app using overlay") }) - }) - - Describe("Using staging security groups", func() { - var testAppName, buildpack string - - BeforeEach(func() { - serverAppName, privateHost, privatePort = pushServerApp() - By("Asserting default staging security group configuration") - testAppName = random_name.CATSRandomName("APP") - buildpack = createDummyBuildpack() - pushApp(testAppName, buildpack) - - privateUri := fmt.Sprintf("%s:%d", privateHost, privatePort) - Expect(cf.Cf("set-env", testAppName, "TESTURI", privateUri).Wait()).To(Exit(0)) - - Expect(cf.Cf("start", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1)) - Eventually(getStagingOutput(testAppName), 5).Should(Say("CURL_EXIT=[^0]"), "Expected staging security groups not to allow internal communication between app containers. Configure your staging security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") + It("validates IPv6 with security groups disabled", func() { + assertAppCanNotConnect() }) - AfterEach(func() { - app_helpers.AppReport(serverAppName) - Expect(cf.Cf("delete", serverAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - app_helpers.AppReport(testAppName) - Expect(cf.Cf("delete", testAppName, "-f", "-r").Wait(Config.CfPushTimeoutDuration())).To(Exit(0)) - - deleteBuildpack(buildpack) + It("binds the wide-open security group", func() { + By("binding the wide-open security group") + bindSecurityGroup(securityGroupName, orgName, spaceName) }) - It("allows external and denies internal traffic during staging based on default IPv6 staging security rules", func() { - Expect(cf.Cf("set-env", testAppName, "TESTURI", "www.ipv6.google.com").Wait()).To(Exit(0)) - Expect(cf.Cf("start", testAppName).Wait(Config.CfPushTimeoutDuration())).To(Exit(1)) - Eventually(getStagingOutput(testAppName), 5).Should(Say("CURL_EXIT=0")) + It("validates IPv6 with security groups enabled", func() { + assertAppCanConnect() }) - }) -}) - -func bindCCSecurityGroupIpv6(orgName, spaceName string) string { - destinations := []Destination{{ - IP: "2001:0db8::/32", // Adjust as necessary for the environment - Ports: "9024", // Adjust as needed for security context - Protocol: "tcp", - }} - securityGroupName := createSecurityGroupIPv6(destinations...) - bindSecurityGroup(securityGroupName, orgName, spaceName) - - return securityGroupName -} - -func createSecurityGroupIPv6(allowedDestinations ...Destination) string { - file, err := os.CreateTemp(os.TempDir(), "CATS-sg-rules") - Expect(err).NotTo(HaveOccurred()) - defer os.Remove(file.Name()) - Expect(json.NewEncoder(file).Encode(allowedDestinations)).To(Succeed()) - - rulesPath := file.Name() - securityGroupName := random_name.CATSRandomName("SG") - - workflowhelpers.AsUser(TestSetup.AdminUserContext(), Config.DefaultTimeoutDuration(), func() { - Expect(cf.Cf("create-security-group", securityGroupName, rulesPath).Wait()).To(Exit(0)) }) - - return securityGroupName -} - -func assertIPv6NetworkingPreconditions(clientAppName string, privateHost string, privatePort int) { - By("Asserting default running security group configuration for traffic between containers") - catnipCurlResponse := testAppConnectivity(clientAppName, privateHost, privatePort) - Expect(catnipCurlResponse.ReturnCode).NotTo(Equal(0), "Expected default running security groups not to allow internal communication between app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") - - By("Asserting default running security group configuration from a running container to an external destination") - catnipCurlResponse = testAppConnectivity(clientAppName, "www.ipv6.google.com", 80) - Expect(catnipCurlResponse.ReturnCode).To(Equal(0), "Expected default running security groups to allow external traffic from app containers. Configure your running security groups to not allow traffic on internal networks, or disable this test by setting 'include_security_groups' to 'false' in '"+os.Getenv("CONFIG")+"'.") -} \ No newline at end of file +}) From ca5759f596e9ea25bea2fb03873f9e495c55f6c9 Mon Sep 17 00:00:00 2001 From: Milena-Encheva <106135092+Milena-Encheva@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:43:50 +0300 Subject: [PATCH 3/8] Delete assets/proxy-ipv6/main.go --- assets/proxy-ipv6/main.go | 115 -------------------------------------- 1 file changed, 115 deletions(-) delete mode 100644 assets/proxy-ipv6/main.go diff --git a/assets/proxy-ipv6/main.go b/assets/proxy-ipv6/main.go deleted file mode 100644 index e922676a2..000000000 --- a/assets/proxy-ipv6/main.go +++ /dev/null @@ -1,115 +0,0 @@ -package main - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io" - "log" - "net" - "net/http" - "os" - "strconv" - "strings" - "time" -) - -func main() { - systemPortString := os.Getenv("PORT") - systemPort, err := strconv.Atoi(systemPortString) - if err != nil { - log.Fatal("Invalid required env var PORT") - } - - mux := http.NewServeMux() - mux.HandleFunc("/proxy/", ipv6ProxyHandler) - mux.HandleFunc("/https_proxy/", ipv6HttpsProxyHandler) - mux.HandleFunc("/", ipv6InfoHandler(systemPort)) - - server := &http.Server{ - Addr: fmt.Sprintf("[::]:%d", systemPort), // Listen on IPv6 interfaces - Handler: mux, - } - _ = server.ListenAndServe() -} - -func ipv6InfoHandler(port int) http.HandlerFunc { - return func(resp http.ResponseWriter, req *http.Request) { - addrs, err := net.InterfaceAddrs() - if err != nil { - panic(err) - } - addressStrings := []string{} - for _, addr := range addrs { - ip, _, _ := net.ParseCIDR(addr.String()) - if ip.To4() == nil { // Ensure it's not IPv4 - addressStrings = append(addressStrings, ip.String()) - } - } - - respBytes, err := json.Marshal(struct { - ListenAddresses []string - Port int - }{ - ListenAddresses: addressStrings, - Port: port, - }) - if err != nil { - panic(err) - } - _, _ = resp.Write(respBytes) - } -} - -func ipv6ProxyHandler(resp http.ResponseWriter, req *http.Request) { - destination := strings.TrimPrefix(req.URL.Path, "/proxy/") - destination = formatIPv6Destination("http", destination) - handleRequest(destination, resp, req) -} - -func ipv6HttpsProxyHandler(resp http.ResponseWriter, req *http.Request) { - destination := strings.TrimPrefix(req.URL.Path, "/https_proxy/") - destination = formatIPv6Destination("https", destination) - handleRequest(destination, resp, req) -} - -func formatIPv6Destination(proto, destination string) string { - if strings.Contains(destination, ":") { - destination = fmt.Sprintf("[%s]", destination) // Encapsulate IPv6 addresses - } - return fmt.Sprintf("%s://%s", proto, destination) -} - -func handleRequest(destination string, resp http.ResponseWriter, req *http.Request) { - getResp, err := httpClient.Get(destination) - if err != nil { - fmt.Fprintf(os.Stderr, "request failed: %s\n", err) - resp.WriteHeader(http.StatusInternalServerError) - _, _ = resp.Write([]byte(fmt.Sprintf("request failed: %s", err))) - return - } - defer getResp.Body.Close() - - readBytes, err := io.ReadAll(getResp.Body) - if err != nil { - resp.WriteHeader(http.StatusInternalServerError) - _, _ = resp.Write([]byte(fmt.Sprintf("read body failed: %s", err))) - return - } - - _, _ = resp.Write(readBytes) -} - -var httpClient = &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - DialContext: (&net.Dialer{ - Timeout: 10 * time.Second, - KeepAlive: 0, - }).DialContext, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, -} - From 98b70f8b28defc52445aec353d57017448cd9614 Mon Sep 17 00:00:00 2001 From: Milena-Encheva <106135092+Milena-Encheva@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:45:38 +0300 Subject: [PATCH 4/8] Delete assets/proxy-ipv6/go.mod --- assets/proxy-ipv6/go.mod | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 assets/proxy-ipv6/go.mod diff --git a/assets/proxy-ipv6/go.mod b/assets/proxy-ipv6/go.mod deleted file mode 100644 index 3d0648ee5..000000000 --- a/assets/proxy-ipv6/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module example-apps/proxy - -go 1.23 - -toolchain go1.23.7 From c93a718be6fa2249083733f24fe92190abb10b8f Mon Sep 17 00:00:00 2001 From: Milena-Encheva <106135092+Milena-Encheva@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:08:18 +0300 Subject: [PATCH 5/8] Delete assets/proxy-ipv6/manifest.yml --- assets/proxy-ipv6/manifest.yml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 assets/proxy-ipv6/manifest.yml diff --git a/assets/proxy-ipv6/manifest.yml b/assets/proxy-ipv6/manifest.yml deleted file mode 100644 index 4b19212cb..000000000 --- a/assets/proxy-ipv6/manifest.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -applications: - - name: proxy - memory: 32M - disk_quota: 32M - buildpacks: - - go_buildpack - env: - GOPACKAGENAME: example-apps/proxy - GOVERSION: go1.23 From 937d9a0ce50c238909ec63aadc939a5fdedf3c6a Mon Sep 17 00:00:00 2001 From: Milena-Encheva <106135092+Milena-Encheva@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:09:29 +0300 Subject: [PATCH 6/8] Delete assets/proxy-ipv6/internal-route-manifest.yml --- assets/proxy-ipv6/internal-route-manifest.yml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 assets/proxy-ipv6/internal-route-manifest.yml diff --git a/assets/proxy-ipv6/internal-route-manifest.yml b/assets/proxy-ipv6/internal-route-manifest.yml deleted file mode 100644 index 0b3a9b937..000000000 --- a/assets/proxy-ipv6/internal-route-manifest.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -applications: - - name: proxy - memory: 32M - disk_quota: 32M - buildpack: go_buildpack - env: - GOPACKAGENAME: example-apps/proxy - GOVERSION: go1.23 - routes: - - route: app-smoke.apps.internal From 56dde73cac199837255e50434e79ae996a2e6a0f Mon Sep 17 00:00:00 2001 From: i759715 Date: Tue, 22 Jul 2025 11:12:32 +0300 Subject: [PATCH 7/8] removed the leftovers for proxy ipv6 package --- helpers/assets/assets.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/helpers/assets/assets.go b/helpers/assets/assets.go index b7728982d..735dc2847 100644 --- a/helpers/assets/assets.go +++ b/helpers/assets/assets.go @@ -32,7 +32,6 @@ type Assets struct { Pora string Php string Proxy string - ProxyIpv6 string R string RubySimple string SecurityGroupBuildpack string @@ -82,7 +81,6 @@ func NewAssets() Assets { Pora: "assets/pora", Php: "assets/php", Proxy: "assets/proxy", - ProxyIpv6: "assets/proxy-ipv6", Python: "assets/python", PythonCrashApp: "assets/python-crash-app", R: "assets/r", From 8a76f74d5cc089ec8e7ca7d9547ae5cbb338662b Mon Sep 17 00:00:00 2001 From: i759715 Date: Tue, 22 Jul 2025 14:41:00 +0300 Subject: [PATCH 8/8] Add the correct link --- security_groups/ipv6_security_groups.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/security_groups/ipv6_security_groups.go b/security_groups/ipv6_security_groups.go index f640149cd..4ba3e628b 100644 --- a/security_groups/ipv6_security_groups.go +++ b/security_groups/ipv6_security_groups.go @@ -44,7 +44,7 @@ var _ = IPv6SecurityGroupsDescribe("IPv6 Security Group", func() { }) assertAppCanConnect := func() { - response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv4-test") + response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv6-test") responseParts := strings.Split(response, "\n") ipAddress := responseParts[0] statusCode := responseParts[1] @@ -55,14 +55,12 @@ var _ = IPv6SecurityGroupsDescribe("IPv6 Security Group", func() { } assertAppCanNotConnect := func() { - response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv4-test") + response := helpers.CurlAppWithStatusCode(Config, appName, "/ipv6-test") responseParts := strings.Split(response, "\n") - ipAddress := responseParts[0] + bodyResponce := responseParts[0] statusCode := responseParts[1] - bodyResponce := net.ParseIP(ipAddress) - Expect(bodyResponce).To(BeNil(), "Expected a non-valid IP address") - Expect(bodyResponce).To(ContainSubstring("not supported by protocol")) + Expect(bodyResponce).To(ContainSubstring("Connection refused")) Expect(statusCode).To(Equal("500")) }