diff --git a/go.mod b/go.mod index c7b9d2cf..8c335ef9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.26.2 require ( github.com/ahmetb/gen-crd-api-reference-docs v0.3.0 + github.com/envoyproxy/go-control-plane/envoy v1.36.0 github.com/gardener/gardener v1.139.2 github.com/gardener/gardener/pkg/apis v1.139.2 github.com/go-logr/logr v1.4.3 @@ -16,7 +17,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 golang.org/x/tools v0.43.0 - gopkg.in/yaml.v3 v3.0.1 + google.golang.org/protobuf v1.36.11 istio.io/api v1.27.8 istio.io/client-go v1.27.2 k8s.io/api v0.35.3 @@ -27,6 +28,7 @@ require ( k8s.io/component-base v0.35.3 k8s.io/utils v0.0.0-20260319190234-28399d86e0b5 sigs.k8s.io/controller-runtime v0.23.3 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -64,11 +66,13 @@ require ( github.com/brunoga/deep v1.3.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/elliotchance/orderedmap/v3 v3.1.0 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fluent/fluent-operator/v3 v3.7.0 // indirect @@ -136,6 +140,7 @@ require ( github.com/perses/common v0.30.2 // indirect github.com/perses/perses v0.53.0 // indirect github.com/perses/perses-operator v0.3.2 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.89.0 // indirect github.com/prometheus/alertmanager v0.29.0 // indirect @@ -202,10 +207,10 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/grpc v1.79.3 // indirect - google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect helm.sh/helm/v3 v3.20.1 // indirect k8s.io/autoscaler/vertical-pod-autoscaler v1.5.1 // indirect k8s.io/gengo v0.0.0-20230829151522-9cce18d56c01 // indirect @@ -222,5 +227,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index ae5f3de4..20fff527 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/Tv1FZd4SCg8axKApyNyRsAt/w= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -134,7 +136,11 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= +github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -411,6 +417,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/pkg/controller/actuator.go b/pkg/controller/actuator.go index 178926a8..5700d732 100644 --- a/pkg/controller/actuator.go +++ b/pkg/controller/actuator.go @@ -276,12 +276,18 @@ func (a *actuator) createSeedResources( return err } - vpnEnvoyFilterSpec := envoyfilters.BuildVPNEnvoyFilterSpecForHelmChart( + vpnEnvoyFilterSpec, err := envoyfilters.BuildVPNEnvoyFilterSpecForHelmChart( cluster, spec.Rule, alwaysAllowedCIDRs, istioLabels, ) - httpProxyEnvoyFilterSpec := envoyfilters.BuildHTTPProxyEnvoyFilterSpecForHelmChart( + if err != nil { + return err + } + httpProxyEnvoyFilterSpec, err := envoyfilters.BuildHTTPProxyEnvoyFilterSpecForHelmChart( cluster, spec.Rule, alwaysAllowedCIDRs, istioLabels, ) + if err != nil { + return err + } cfg := map[string]interface{}{ "shootName": cluster.Shoot.Status.TechnicalID, @@ -298,8 +304,11 @@ func (a *actuator) createSeedResources( // The `nginx-ingress-controller` Gateway object only exists in g/g@v1.89, (introduced with // https://github.com/gardener/gardener/pull/9038). // If it doesn't exist yet, we can't apply ACLs to shoot ingresses. - ingressEnvoyFilterSpec := envoyfilters.BuildIngressEnvoyFilterSpecForHelmChart( + ingressEnvoyFilterSpec, err := envoyfilters.BuildIngressEnvoyFilterSpecForHelmChart( cluster, spec.Rule, alwaysAllowedCIDRs, defaultLabels) + if err != nil { + return err + } cfg["ingressEnvoyFilterSpec"] = ingressEnvoyFilterSpec } diff --git a/pkg/envoyfilters/envoyfilters.go b/pkg/envoyfilters/envoyfilters.go index 5ceca13d..8bad7ce8 100644 --- a/pkg/envoyfilters/envoyfilters.go +++ b/pkg/envoyfilters/envoyfilters.go @@ -6,7 +6,18 @@ import ( "net" "strings" + envoy_corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_rbacv3 "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + envoy_routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + envoy_httprbacv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + envoy_networkrbacv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/rbac/v3" + envoy_matcherv3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" "github.com/gardener/gardener/extensions/pkg/controller" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" + istio_networkingv1alpha3 "istio.io/api/networking/v1alpha3" "github.com/stackitcloud/gardener-extension-acl/pkg/helper" ) @@ -27,21 +38,50 @@ type ACLRule struct { Type string `json:"type"` } +func (r *ACLRule) actionProto() (envoy_rbacv3.RBAC_Action, error) { + switch r.Action { + case "DENY": + return envoy_rbacv3.RBAC_DENY, nil + case "ALLOW": + return envoy_rbacv3.RBAC_ALLOW, nil + default: + return -1, fmt.Errorf("unknown action %s", r.Action) + } +} + +// FilterPatch represents the object beneath EnvoyFilter.spec.configPatches.patch.value +// It holds the name of the filter and it's typed config to inject into the envoy config +type FilterPatch struct { + Name string `json:"name"` + TypedConfig *structpb.Struct `json:"typed_config"` +} + +// asStructPB returns FilterPatch represented as a structpb.Struct +func (f *FilterPatch) asStructPB() (*structpb.Struct, error) { + pb, err := structpb.NewStruct(map[string]any{ + "name": f.Name, + "typed_config": f.TypedConfig.AsMap(), + }) + if err != nil { + return nil, err + } + return pb, nil +} + // BuildAPIEnvoyFilterSpecForHelmChart assembles EnvoyFilter patches for API server // networking for every rule in the extension spec. func BuildAPIEnvoyFilterSpecForHelmChart( rule *ACLRule, hosts, alwaysAllowedCIDRs []string, istioLabels map[string]string, -) (map[string]interface{}, error) { - apiConfigPatch, err := CreateAPIConfigPatchFromRule(rule, hosts, alwaysAllowedCIDRs) +) (*istio_networkingv1alpha3.EnvoyFilter, error) { + apiConfigPatch, err := createAPIConfigPatchFromRule(rule, hosts, alwaysAllowedCIDRs) if err != nil { return nil, err } - - return map[string]interface{}{ - "workloadSelector": map[string]interface{}{ - "labels": istioLabels, + return &istio_networkingv1alpha3.EnvoyFilter{ + WorkloadSelector: &istio_networkingv1alpha3.WorkloadSelector{ + Labels: istioLabels, }, - "configPatches": []map[string]interface{}{ + ConfigPatches: []*istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ apiConfigPatch, }, }, nil @@ -51,27 +91,28 @@ func BuildAPIEnvoyFilterSpecForHelmChart( // endpoints using the seed ingress domain. func BuildIngressEnvoyFilterSpecForHelmChart( cluster *controller.Cluster, rule *ACLRule, alwaysAllowedCIDRs []string, istioLabels map[string]string, -) map[string]interface{} { +) (*istio_networkingv1alpha3.EnvoyFilter, error) { seedIngressDomain := helper.GetSeedIngressDomain(cluster.Seed) - if seedIngressDomain != "" { - shootID := helper.ComputeShortShootID(cluster.Shoot) - - return map[string]interface{}{ - "workloadSelector": map[string]interface{}{ - "labels": istioLabels, - }, - "configPatches": []map[string]interface{}{ - CreateIngressConfigPatchFromRule(rule, seedIngressDomain, shootID, alwaysAllowedCIDRs), - }, - } + if seedIngressDomain == "" { + return nil, nil } - return nil + shootID := helper.ComputeShortShootID(cluster.Shoot) + patch, err := createIngressConfigPatchFromRule(rule, seedIngressDomain, shootID, alwaysAllowedCIDRs) + if err != nil { + return nil, err + } + return &istio_networkingv1alpha3.EnvoyFilter{ + WorkloadSelector: &istio_networkingv1alpha3.WorkloadSelector{ + Labels: istioLabels, + }, + ConfigPatches: []*istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{patch}, + }, nil } // BuildVPNEnvoyFilterSpecForHelmChart assembles EnvoyFilter patches for VPN. func BuildVPNEnvoyFilterSpecForHelmChart( cluster *controller.Cluster, rule *ACLRule, alwaysAllowedCIDRs []string, istioLabels map[string]string, -) map[string]interface{} { +) (*istio_networkingv1alpha3.EnvoyFilter, error) { return buildProxyEnvoyFilterSpecForHelmChart(httpProxyFilterOptions{ Rule: rule, ShortShootID: helper.ComputeShortShootID(cluster.Shoot), @@ -88,7 +129,7 @@ func BuildVPNEnvoyFilterSpecForHelmChart( // BuildHTTPProxyEnvoyFilterSpecForHelmChart assembles EnvoyFilter patches for the unified HTTP proxy port. func BuildHTTPProxyEnvoyFilterSpecForHelmChart( cluster *controller.Cluster, rule *ACLRule, alwaysAllowedCIDRs []string, istioLabels map[string]string, -) map[string]interface{} { +) (*istio_networkingv1alpha3.EnvoyFilter, error) { return buildProxyEnvoyFilterSpecForHelmChart(httpProxyFilterOptions{ Rule: rule, ShortShootID: helper.ComputeShortShootID(cluster.Shoot), @@ -102,94 +143,129 @@ func BuildHTTPProxyEnvoyFilterSpecForHelmChart( }) } -// CreateAPIConfigPatchFromRule combines an ACLRule, the first entry of the +// createAPIConfigPatchFromRule combines an ACLRule, the first entry of the // hosts list and the alwaysAllowedCIDRs into a network filter patch that can be // applied to the `GATEWAY` network filter chain matching the host. -func CreateAPIConfigPatchFromRule( +func createAPIConfigPatchFromRule( rule *ACLRule, hosts, alwaysAllowedCIDRs []string, -) (map[string]interface{}, error) { +) (*istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch, error) { if len(hosts) == 0 { return nil, ErrNoHostsGiven } rbacName := "acl-api" + action, err := rule.actionProto() + if err != nil { + return nil, err + } principals := ruleCIDRsToPrincipal(rule, alwaysAllowedCIDRs) + principalPatch, err := principalsToPatch(rbacName, action, principals) + if err != nil { + return nil, err + } - return map[string]interface{}{ - "applyTo": "NETWORK_FILTER", - "match": map[string]interface{}{ - "context": "GATEWAY", - "listener": map[string]interface{}{ - "filterChain": map[string]interface{}{ - // There is one filter chain per shoot in the SNI listener that has two SNI matches: one for the internal and - // one for the external shoot domain. - // We can use either shoot domain to match the filter chain that we want to patch with this EnvoyFilter. - // The ACL config will apply to traffic going via both the internal and the external API server address. - // See: https://istio.io/latest/docs/reference/config/networking/envoy-filter/#EnvoyFilter-ListenerMatch-FilterChainMatch - "sni": hosts[0], + return &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: istio_networkingv1alpha3.EnvoyFilter_NETWORK_FILTER, + Match: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: istio_networkingv1alpha3.EnvoyFilter_GATEWAY, + ObjectTypes: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &istio_networkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &istio_networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + // There is one filter chain per shoot in the SNI listener that has two SNI matches: one for the internal and + // one for the external shoot domain. + // We can use either shoot domain to match the filter chain that we want to patch with this EnvoyFilter. + // The ACL config will apply to traffic going via both the internal and the external API server address. + // See: https://istio.io/latest/docs/reference/config/networking/envoy-filter/#EnvoyFilter-ListenerMatch-FilterChainMatch + Sni: hosts[0], + }, }, }, }, - "patch": principalsToPatch(rbacName, rule.Action, "network", principals), + Patch: principalPatch, }, nil } -// CreateIngressConfigPatchFromRule creates a network filter patch that can be -// applied to the `GATEWAY` network filter chain matching the wildcard ingress domain. -func CreateIngressConfigPatchFromRule( +func createIngressConfigPatchFromRule( rule *ACLRule, seedIngressDomain, shootID string, alwaysAllowedCIDRs []string, -) map[string]interface{} { +) (*istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch, error) { rbacName := "acl-ingress" ingressSuffix := "-" + shootID + "." + seedIngressDomain - return map[string]interface{}{ - "applyTo": "NETWORK_FILTER", - "match": map[string]interface{}{ - "context": "GATEWAY", - "listener": map[string]interface{}{ - "filterChain": map[string]interface{}{ - "sni": "*." + seedIngressDomain, - }, + + requestedServerNameRule := &envoy_rbacv3.Permission_RequestedServerName{ + RequestedServerName: &envoy_matcherv3.StringMatcher{ + MatchPattern: &envoy_matcherv3.StringMatcher_Suffix{ + Suffix: ingressSuffix, }, }, + } - "patch": map[string]interface{}{ - "operation": "INSERT_FIRST", - "value": map[string]interface{}{ - "name": rbacName, - "typed_config": map[string]interface{}{ - "@type": "type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC", - "rules": map[string]interface{}{ - "action": "ALLOW", - "policies": map[string]interface{}{ - shootID + "-inverse": map[string]interface{}{ - "permissions": []map[string]interface{}{{ - "not_rule": map[string]interface{}{ - "requested_server_name": map[string]interface{}{ - "suffix": ingressSuffix, - }, - }, - }}, - "principals": []map[string]interface{}{{ - "remote_ip": map[string]interface{}{ - "address_prefix": "0.0.0.0", - "prefix_len": 0, - }, - }}, + rbacFilter := &envoy_networkrbacv3.RBAC{ + Rules: &envoy_rbacv3.RBAC{ + Action: envoy_rbacv3.RBAC_ALLOW, + Policies: map[string]*envoy_rbacv3.Policy{ + shootID + "-inverse": { + Permissions: []*envoy_rbacv3.Permission{ + { + Rule: &envoy_rbacv3.Permission_NotRule{ + NotRule: &envoy_rbacv3.Permission{ + Rule: requestedServerNameRule, + }, }, - shootID: map[string]interface{}{ - "permissions": []map[string]interface{}{{ - "requested_server_name": map[string]interface{}{ - "suffix": ingressSuffix, - }, - }}, - "principals": ruleCIDRsToPrincipal(rule, alwaysAllowedCIDRs), + }, + }, + Principals: []*envoy_rbacv3.Principal{ + { + Identifier: &envoy_rbacv3.Principal_RemoteIp{ + RemoteIp: &envoy_corev3.CidrRange{ + AddressPrefix: "0.0.0.0", + PrefixLen: wrapperspb.UInt32(0), + }, }, }, }, - "stat_prefix": "envoyrbac", + }, + shootID: { + Principals: ruleCIDRsToPrincipal(rule, alwaysAllowedCIDRs), + Permissions: []*envoy_rbacv3.Permission{ + { + Rule: requestedServerNameRule, + }, + }, }, }, }, + StatPrefix: "envoyrbac", } + + typedConfig, err := protoMessageToTypedConfig(rbacFilter) + if err != nil { + return nil, err + } + filterPatch := &FilterPatch{ + Name: rbacName, + TypedConfig: typedConfig, + } + pb, err := filterPatch.asStructPB() + if err != nil { + return nil, err + } + + return &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: istio_networkingv1alpha3.EnvoyFilter_NETWORK_FILTER, + Match: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: istio_networkingv1alpha3.EnvoyFilter_GATEWAY, + ObjectTypes: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &istio_networkingv1alpha3.EnvoyFilter_ListenerMatch{ + FilterChain: &istio_networkingv1alpha3.EnvoyFilter_ListenerMatch_FilterChainMatch{ + Sni: "*." + seedIngressDomain, + }, + }, + }, + }, + Patch: &istio_networkingv1alpha3.EnvoyFilter_Patch{ + Operation: istio_networkingv1alpha3.EnvoyFilter_Patch_INSERT_FIRST, + Value: pb, + }, + }, nil } type httpProxyFilterOptions struct { @@ -203,11 +279,11 @@ type httpProxyFilterOptions struct { Port uint32 } -func buildProxyEnvoyFilterSpecForHelmChart(p httpProxyFilterOptions) map[string]interface{} { +func buildProxyEnvoyFilterSpecForHelmChart(p httpProxyFilterOptions) (*istio_networkingv1alpha3.EnvoyFilter, error) { rbacName := "acl" + p.NameSuffix - headerMatcher := map[string]interface{}{ - "name": p.Header, - "string_match": map[string]interface{}{ + headerMatcher := envoy_routev3.HeaderMatcher{ + Name: p.Header, + HeaderMatchSpecifier: &envoy_routev3.HeaderMatcher_StringMatch{ // The actual header value will look something like // `outbound|1194||vpn-seed-server..svc.cluster.local`. // Include dots in the contains matcher as anchors, to always match the entire technical shoot ID. @@ -215,76 +291,90 @@ func buildProxyEnvoyFilterSpecForHelmChart(p httpProxyFilterOptions) map[string] // `foo` would effectively inherit the ACL of `foo-bar`. // We don't match with the full header value to allow service names and ports to change while still making sure // we catch all traffic targeting this shoot. - "contains": "." + p.TechnicalShootID + ".", - }, - } - configPatch := map[string]interface{}{ - "applyTo": "HTTP_FILTER", - "match": map[string]interface{}{ - "context": "GATEWAY", - "listener": map[string]interface{}{ - "name": fmt.Sprintf("0.0.0.0_%d", p.Port), + StringMatch: &envoy_matcherv3.StringMatcher{ + MatchPattern: &envoy_matcherv3.StringMatcher_Contains{ + Contains: "." + p.TechnicalShootID + ".", + }, }, }, - "patch": map[string]interface{}{ - "operation": "INSERT_FIRST", - "value": map[string]interface{}{ - "name": rbacName, - "typed_config": map[string]interface{}{ - "@type": "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", - "rules": map[string]interface{}{ - "action": "ALLOW", - "policies": map[string]interface{}{ - p.ShortShootID + "-inverse": map[string]interface{}{ - "permissions": []map[string]interface{}{{ - "not_rule": map[string]interface{}{ - "header": headerMatcher, - }, - }}, - "principals": []map[string]interface{}{{ - "remote_ip": map[string]interface{}{ - "address_prefix": "0.0.0.0", - "prefix_len": 0, + } + rbacFilter := &envoy_httprbacv3.RBAC{ + RulesStatPrefix: "envoyrbac", + Rules: &envoy_rbacv3.RBAC{ + Action: envoy_rbacv3.RBAC_ALLOW, + Policies: map[string]*envoy_rbacv3.Policy{ + p.ShortShootID + "-inverse": { + Permissions: []*envoy_rbacv3.Permission{ + { + Rule: &envoy_rbacv3.Permission_NotRule{ + NotRule: &envoy_rbacv3.Permission{ + Rule: &envoy_rbacv3.Permission_Header{ + Header: &headerMatcher, }, - }}, + }, }, - p.ShortShootID: map[string]interface{}{ - "permissions": []map[string]interface{}{{ - "header": headerMatcher, - }}, - "principals": ruleCIDRsToPrincipal(p.Rule, p.AlwaysAllowedCIDRs), + }, + }, + Principals: []*envoy_rbacv3.Principal{ + { + Identifier: &envoy_rbacv3.Principal_RemoteIp{ + RemoteIp: &envoy_corev3.CidrRange{ + AddressPrefix: "0.0.0.0", + PrefixLen: wrapperspb.UInt32(0), + }, }, }, }, - "stat_prefix": "envoyrbac", + }, + p.ShortShootID: { + Permissions: []*envoy_rbacv3.Permission{ + { + Rule: &envoy_rbacv3.Permission_Header{ + Header: &headerMatcher, + }, + }, + }, + Principals: ruleCIDRsToPrincipal(p.Rule, p.AlwaysAllowedCIDRs), }, }, }, } + typedConfig, err := protoMessageToTypedConfig(rbacFilter) + if err != nil { + return nil, err + } + filterPatch := &FilterPatch{ + Name: rbacName, + TypedConfig: typedConfig, + } + pb, err := filterPatch.asStructPB() + if err != nil { + return nil, err + } - return map[string]interface{}{ - "workloadSelector": map[string]interface{}{ - "labels": p.IstioLabels, + configPatch := &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: istio_networkingv1alpha3.EnvoyFilter_HTTP_FILTER, + Match: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: istio_networkingv1alpha3.EnvoyFilter_GATEWAY, + ObjectTypes: &istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectMatch_Listener{ + Listener: &istio_networkingv1alpha3.EnvoyFilter_ListenerMatch{ + Name: fmt.Sprintf("0.0.0.0_%d", p.Port), + }, + }, }, - "configPatches": []map[string]interface{}{ - configPatch, + Patch: &istio_networkingv1alpha3.EnvoyFilter_Patch{ + Operation: istio_networkingv1alpha3.EnvoyFilter_Patch_INSERT_FIRST, + Value: pb, }, } -} -// CreateInternalFilterPatchFromRule combines an ACLRule, the -// alwaysAllowedCIDRs, and the shootSpecificCIDRs into a filter patch. -func CreateInternalFilterPatchFromRule( - rule *ACLRule, - alwaysAllowedCIDRs []string, - shootSpecificCIDRs []string, -) (map[string]interface{}, error) { - rbacName := "acl-internal" - principals := ruleCIDRsToPrincipal(rule, append(alwaysAllowedCIDRs, shootSpecificCIDRs...)) - - return map[string]interface{}{ - "name": rbacName + "-" + strings.ToLower(rule.Type), - "typed_config": typedConfigToPatch(rbacName, rule.Action, "network", principals), + return &istio_networkingv1alpha3.EnvoyFilter{ + WorkloadSelector: &istio_networkingv1alpha3.WorkloadSelector{ + Labels: p.IstioLabels, + }, + ConfigPatches: []*istio_networkingv1alpha3.EnvoyFilter_EnvoyConfigObjectPatch{ + configPatch, + }, }, nil } @@ -292,20 +382,30 @@ func CreateInternalFilterPatchFromRule( // into a list of envoy principals. The function checks for the rule action: If // the action is "ALLOW", the alwaysAllowedCIDRs are appended to the principals // to guarantee the downstream flow for these CIDRs is not blocked. -func ruleCIDRsToPrincipal(rule *ACLRule, alwaysAllowedCIDRs []string) []map[string]interface{} { - principals := []map[string]interface{}{} +func ruleCIDRsToPrincipal(rule *ACLRule, alwaysAllowedCIDRs []string) []*envoy_rbacv3.Principal { + principals := []*envoy_rbacv3.Principal{} for _, cidr := range rule.Cidrs { prefix, length, err := getPrefixAndPrefixLength(cidr) if err != nil { continue } - principals = append(principals, map[string]interface{}{ - strings.ToLower(rule.Type): map[string]interface{}{ - "address_prefix": prefix, - "prefix_len": length, - }, - }) + cidrRange := envoy_corev3.CidrRange{ + AddressPrefix: prefix, + PrefixLen: wrapperspb.UInt32(uint32(length)), + } + p := new(envoy_rbacv3.Principal) + switch strings.ToLower(rule.Type) { + case "source_ip": + p.Identifier = &envoy_rbacv3.Principal_SourceIp{SourceIp: &cidrRange} + case "remote_ip": + p.Identifier = &envoy_rbacv3.Principal_RemoteIp{RemoteIp: &cidrRange} + case "direct_remote_ip": + p.Identifier = &envoy_rbacv3.Principal_DirectRemoteIp{DirectRemoteIp: &cidrRange} + default: + continue + } + principals = append(principals, p) } // if the rule has action "ALLOW" (which means "limit the access to only the @@ -317,10 +417,12 @@ func ruleCIDRsToPrincipal(rule *ACLRule, alwaysAllowedCIDRs []string) []map[stri if err != nil { continue } - principals = append(principals, map[string]interface{}{ - "remote_ip": map[string]interface{}{ - "address_prefix": prefix, - "prefix_len": length, + principals = append(principals, &envoy_rbacv3.Principal{ + Identifier: &envoy_rbacv3.Principal_RemoteIp{ + RemoteIp: &envoy_corev3.CidrRange{ + AddressPrefix: prefix, + PrefixLen: wrapperspb.UInt32(uint32(length)), + }, }, }) } @@ -342,31 +444,60 @@ func getPrefixAndPrefixLength(cidr string) (prefix string, prefixLen int, err er } func principalsToPatch( - rbacName, ruleAction, filterType string, principals []map[string]interface{}, -) map[string]interface{} { - return map[string]interface{}{ - "operation": "INSERT_FIRST", - "value": map[string]interface{}{ - "name": rbacName, - "typed_config": typedConfigToPatch(rbacName, ruleAction, filterType, principals), - }, + rbacName string, ruleAction envoy_rbacv3.RBAC_Action, principals []*envoy_rbacv3.Principal, +) (*istio_networkingv1alpha3.EnvoyFilter_Patch, error) { + rbacFilter := newRBACFilter(rbacName, ruleAction, principals) + typedConfig, err := protoMessageToTypedConfig(rbacFilter) + if err != nil { + return nil, err + } + filter := &FilterPatch{ + Name: rbacName, + TypedConfig: typedConfig, } + pb, err := filter.asStructPB() + if err != nil { + return nil, err + } + return &istio_networkingv1alpha3.EnvoyFilter_Patch{ + Operation: istio_networkingv1alpha3.EnvoyFilter_Patch_INSERT_FIRST, + Value: pb, + }, nil } -func typedConfigToPatch(rbacName, ruleAction, filterType string, principals []map[string]interface{}) map[string]interface{} { - return map[string]interface{}{ - "@type": "type.googleapis.com/envoy.extensions.filters." + filterType + ".rbac.v3.RBAC", - "stat_prefix": "envoyrbac", - "rules": map[string]interface{}{ - "action": strings.ToUpper(ruleAction), - "policies": map[string]interface{}{ - rbacName: map[string]interface{}{ - "permissions": []map[string]interface{}{ - {"any": true}, +func newRBACFilter(rbacName string, ruleAction envoy_rbacv3.RBAC_Action, principals []*envoy_rbacv3.Principal) *envoy_networkrbacv3.RBAC { + return &envoy_networkrbacv3.RBAC{ + StatPrefix: "envoyrbac", + Rules: &envoy_rbacv3.RBAC{ + Action: ruleAction, + Policies: map[string]*envoy_rbacv3.Policy{ + rbacName: { + Permissions: []*envoy_rbacv3.Permission{ + { + Rule: &envoy_rbacv3.Permission_Any{ + Any: true, + }, + }, }, - "principals": principals, + Principals: principals, }, }, }, } } + +func protoMessageToTypedConfig(m proto.Message) (*structpb.Struct, error) { + raw, err := protojson.MarshalOptions{ + UseProtoNames: true, + }.Marshal(m) + if err != nil { + return nil, err + } + s := new(structpb.Struct) + if err := protojson.Unmarshal(raw, s); err != nil { + return nil, err + } + typeName := fmt.Sprintf("type.googleapis.com/%s", proto.MessageName(m)) + s.Fields["@type"] = structpb.NewStringValue(typeName) + return s, nil +} diff --git a/pkg/envoyfilters/envoyfilters_test.go b/pkg/envoyfilters/envoyfilters_test.go index b3ece24a..4f05f790 100644 --- a/pkg/envoyfilters/envoyfilters_test.go +++ b/pkg/envoyfilters/envoyfilters_test.go @@ -1,6 +1,7 @@ package envoyfilters import ( + "encoding/json" "os" "path" @@ -8,8 +9,8 @@ import ( "github.com/gardener/gardener/pkg/extensions" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/yaml" ) var _ = Describe("EnvoyFilter Unit Tests", func() { @@ -52,7 +53,7 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { result, err := BuildAPIEnvoyFilterSpecForHelmChart(rule, hosts, alwaysAllowedCIDRs, labels) Expect(err).ToNot(HaveOccurred()) - checkIfMapEqualsYAML(result, "apiEnvoyFilterSpecWithOneAllowRule.yaml") + checkIfFilterEquals(result, "apiEnvoyFilterSpecWithOneAllowRule.yaml") }) }) }) @@ -65,9 +66,10 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { "app": "istio-ingressgateway", "istio": "ingressgateway", } - ingressEnvoyFilterSpec := BuildIngressEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + ingressEnvoyFilterSpec, err := BuildIngressEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + Expect(err).NotTo(HaveOccurred()) - checkIfMapEqualsYAML(ingressEnvoyFilterSpec, "ingressEnvoyFilterSpecWithOneAllowRule.yaml") + checkIfFilterEquals(ingressEnvoyFilterSpec, "ingressEnvoyFilterSpecWithOneAllowRule.yaml") }) It("Should not create an envoyFilter spec when seed has no ingress", func() { rule := createRule("ALLOW", "remote_ip", "10.180.0.0/16") @@ -76,8 +78,9 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { "app": "istio-ingressgateway", "istio": "ingressgateway", } - ingressEnvoyFilterSpec := BuildIngressEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) - Expect(ingressEnvoyFilterSpec["ingressEnvoyFilterSpec"]).To(BeNil()) + ingressEnvoyFilterSpec, err := BuildIngressEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + Expect(err).NotTo(HaveOccurred()) + Expect(ingressEnvoyFilterSpec).To(BeNil()) }) }) }) @@ -90,9 +93,10 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { "app": "istio-ingressgateway", "istio": "ingressgateway", } - result := BuildVPNEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + result, err := BuildVPNEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + Expect(err).NotTo(HaveOccurred()) - checkIfMapEqualsYAML(result, "vpnEnvoyFilterSpecWithOneAllowRule.yaml") + checkIfFilterEquals(result, "vpnEnvoyFilterSpecWithOneAllowRule.yaml") }) }) }) @@ -105,22 +109,10 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { "app": "istio-ingressgateway", "istio": "ingressgateway", } - result := BuildHTTPProxyEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + result, err := BuildHTTPProxyEnvoyFilterSpecForHelmChart(cluster, rule, alwaysAllowedCIDRs, labels) + Expect(err).NotTo(HaveOccurred()) - checkIfMapEqualsYAML(result, "httpProxyEnvoyFilterSpecWithOneAllowRule.yaml") - }) - }) - }) - - Describe("CreateInternalFilterPatchFromRule", func() { - When("there is an allow rule", func() { - It("Should create a filter spec matching the expected one, including the always allowed CIDRs", func() { - rule := createRule("ALLOW", "remote_ip", "0.0.0.0/0") - - result, err := CreateInternalFilterPatchFromRule(rule, alwaysAllowedCIDRs, []string{}) - - Expect(err).ToNot(HaveOccurred()) - checkIfMapEqualsYAML(result, "singleFiltersAllowEntry.yaml") + checkIfFilterEquals(result, "httpProxyEnvoyFilterSpecWithOneAllowRule.yaml") }) }) }) @@ -130,7 +122,7 @@ var _ = Describe("EnvoyFilter Unit Tests", func() { It("should return the appropriate error", func() { rule := createRule("ALLOW", "remote_ip", "0.0.0.0/0") - result, err := CreateAPIConfigPatchFromRule(rule, nil, alwaysAllowedCIDRs) + result, err := createAPIConfigPatchFromRule(rule, nil, alwaysAllowedCIDRs) Expect(err).To(Equal(ErrNoHostsGiven)) Expect(result).To(BeNil()) @@ -150,19 +142,19 @@ func createRule(action, ruleType, cidr string) *ACLRule { } } -// checkIfMapEqualsYAML takes a map as input, and tries to compare its +// checkIfFilterEquals takes a map or proto message as input, and tries to compare its // marshaled contents to the string coming from the specified testdata file. // Fails the test if strings differ. The file contents are unmarshaled and // marshaled again to guarantee the strings are comparable. -func checkIfMapEqualsYAML(input map[string]interface{}, relTestingFilePath string) { - goldenYAMLByteArray, err := os.ReadFile(path.Join("./testdata", relTestingFilePath)) +func checkIfFilterEquals(input any, relTestingFilePath string) { + goldenYAMLBytes, err := os.ReadFile(path.Join("./testdata", relTestingFilePath)) Expect(err).ToNot(HaveOccurred()) - goldenMap := map[string]interface{}{} - Expect(yaml.Unmarshal(goldenYAMLByteArray, goldenMap)).To(Succeed()) - goldenYAMLProcessedByteArray, err := yaml.Marshal(goldenMap) + + goldenJSON, err := yaml.YAMLToJSON(goldenYAMLBytes) Expect(err).ToNot(HaveOccurred()) - inputByteArray, err := yaml.Marshal(input) + actual, err := json.Marshal(input) Expect(err).ToNot(HaveOccurred()) - Expect(string(inputByteArray)).To(Equal(string(goldenYAMLProcessedByteArray))) + + Expect(actual).To(MatchJSON(goldenJSON)) } diff --git a/pkg/envoyfilters/testdata/apiEnvoyFilterSpecWithOneAllowRule.yaml b/pkg/envoyfilters/testdata/apiEnvoyFilterSpecWithOneAllowRule.yaml index 85ffc768..1f5277d0 100644 --- a/pkg/envoyfilters/testdata/apiEnvoyFilterSpecWithOneAllowRule.yaml +++ b/pkg/envoyfilters/testdata/apiEnvoyFilterSpecWithOneAllowRule.yaml @@ -12,7 +12,7 @@ configPatches: typed_config: '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: acl-api: permissions: @@ -31,4 +31,4 @@ configPatches: workloadSelector: labels: app: istio-ingressgateway - istio: ingressgateway \ No newline at end of file + istio: ingressgateway diff --git a/pkg/envoyfilters/testdata/httpProxyEnvoyFilterSpecWithOneAllowRule.yaml b/pkg/envoyfilters/testdata/httpProxyEnvoyFilterSpecWithOneAllowRule.yaml index cd1bb117..5468392c 100644 --- a/pkg/envoyfilters/testdata/httpProxyEnvoyFilterSpecWithOneAllowRule.yaml +++ b/pkg/envoyfilters/testdata/httpProxyEnvoyFilterSpecWithOneAllowRule.yaml @@ -11,7 +11,7 @@ configPatches: typed_config: '@type': type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: bar--foo-inverse: permissions: @@ -40,7 +40,7 @@ configPatches: - remote_ip: address_prefix: 10.96.0.0 prefix_len: 11 - stat_prefix: envoyrbac + rules_stat_prefix: envoyrbac workloadSelector: labels: app: istio-ingressgateway diff --git a/pkg/envoyfilters/testdata/ingressEnvoyFilterSpecWithOneAllowRule.yaml b/pkg/envoyfilters/testdata/ingressEnvoyFilterSpecWithOneAllowRule.yaml index 0a0f9442..8e6d00a6 100644 --- a/pkg/envoyfilters/testdata/ingressEnvoyFilterSpecWithOneAllowRule.yaml +++ b/pkg/envoyfilters/testdata/ingressEnvoyFilterSpecWithOneAllowRule.yaml @@ -12,7 +12,7 @@ configPatches: typed_config: '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: bar--foo: permissions: diff --git a/pkg/envoyfilters/testdata/legacyVPNEnvoyFilterSpecWithOneAllowRule.yaml b/pkg/envoyfilters/testdata/legacyVPNEnvoyFilterSpecWithOneAllowRule.yaml index 76081512..b2e9fcde 100644 --- a/pkg/envoyfilters/testdata/legacyVPNEnvoyFilterSpecWithOneAllowRule.yaml +++ b/pkg/envoyfilters/testdata/legacyVPNEnvoyFilterSpecWithOneAllowRule.yaml @@ -11,7 +11,7 @@ configPatches: typed_config: '@type': type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: acl-vpn-inverse: permissions: @@ -45,7 +45,7 @@ configPatches: name: reversed-vpn string_match: contains: .shoot--projectname--shootname. - stat_prefix: envoyrbac + rules_stat_prefix: envoyrbac workloadSelector: labels: app: istio-ingressgateway diff --git a/pkg/envoyfilters/testdata/singleFiltersAllowEntry.yaml b/pkg/envoyfilters/testdata/singleFiltersAllowEntry.yaml index feb69582..b1c2e2f8 100644 --- a/pkg/envoyfilters/testdata/singleFiltersAllowEntry.yaml +++ b/pkg/envoyfilters/testdata/singleFiltersAllowEntry.yaml @@ -2,7 +2,7 @@ name: acl-internal-remote_ip typed_config: '@type': type.googleapis.com/envoy.extensions.filters.network.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: acl-internal: permissions: diff --git a/pkg/envoyfilters/testdata/vpnEnvoyFilterSpecWithOneAllowRule.yaml b/pkg/envoyfilters/testdata/vpnEnvoyFilterSpecWithOneAllowRule.yaml index 69617728..c4fa9154 100644 --- a/pkg/envoyfilters/testdata/vpnEnvoyFilterSpecWithOneAllowRule.yaml +++ b/pkg/envoyfilters/testdata/vpnEnvoyFilterSpecWithOneAllowRule.yaml @@ -11,7 +11,7 @@ configPatches: typed_config: '@type': type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC rules: - action: ALLOW + # action: ALLOW # allow is not being marshaled by protojson as it is the default enum value (0) policies: bar--foo-inverse: permissions: @@ -40,7 +40,7 @@ configPatches: - remote_ip: address_prefix: 10.96.0.0 prefix_len: 11 - stat_prefix: envoyrbac + rules_stat_prefix: envoyrbac workloadSelector: labels: app: istio-ingressgateway