From ef04f3e790e20082f83c49195031d3b259b57ab7 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sat, 4 Apr 2026 12:18:02 +0200 Subject: [PATCH 01/14] build: fix Signed-off-by: Jarno Rajahalme --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 446e8b225..f29e5edb5 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ ifdef BAZEL_REMOTE_CACHE BAZEL_BUILD_OPTS += --remote_cache=$(BAZEL_REMOTE_CACHE) endif -BAZEL_TEST_OPTS ?= --jobs=HOST_RAM*.0003 --test_timeout=300 --local_test_jobs=1 --flaky_test_attempts=3 +BAZEL_TEST_OPTS ?= --jobs=HOST_RAM*.0002 --test_timeout=100 --local_test_jobs=1 --flaky_test_attempts=3 BAZEL_TEST_OPTS += --test_output=errors BUILDARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(shell uname -m))) From 99a326a9ef213f5652b8f4bd6f72011e2bb29203 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 2 Apr 2026 15:41:45 +0200 Subject: [PATCH 02/14] policy: Remove memory leak on exception Keep the new map in the stack so that it is not leaked on exception. Move-construct to the heap when exchanging for the old map. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index e3a460cbd..ba9127f8a 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -1961,8 +1961,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( transport_factory_context_->setInitManager(version_init_manager); const auto* old_map = load(); + RawPolicyMap new_map; { - auto new_map = new RawPolicyMap(); try { for (const auto& resource : resources) { const auto& config = dynamic_cast(resource.get().resource()); @@ -1984,7 +1984,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ENVOY_LOG(trace, "New policy is equal to old one, not updating."); for (const auto& endpoint_ip : config.endpoint_ips()) { ENVOY_LOG(trace, "Cilium keeping network policy for endpoint {}", endpoint_ip); - new_map->emplace(endpoint_ip, old_policy); + new_map.emplace(endpoint_ip, old_policy); } continue; } @@ -1996,7 +1996,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( for (const auto& endpoint_ip : config.endpoint_ips()) { ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); // new_map is not exception safe, new_policy must be computed separately! - new_map->emplace(endpoint_ip, new_policy); + new_map.emplace(endpoint_ip, new_policy); } } } catch (const EnvoyException& e) { @@ -2013,7 +2013,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // Swap the new map in, new_map goes out of scope right after to eliminate accidental // modification. - old_map = exchange(new_map); + old_map = exchange(new RawPolicyMap(std::move(new_map))); } // Delete the old map once all worker threads have entered their event queues, as this From 635ed25113cdd18d7d376186eb5ec3599f8c245b Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 2 Apr 2026 11:47:49 +0200 Subject: [PATCH 03/14] policy: Refactor for future changes Refactor policy code in preparation for delta NPDS changes in later commits: - Remove unnecessary PortPolicy.map_ member. - Move NetworkPolicyMapImpl to cilium/network_policy.cc to hide implementation detail. - Add helpers for policy updates to be shared with the delta code later. - Pass Context to secret watcher functions instead of the full NetworkPolicyMap. - Simplify gRPC subscribe() parameters. Signed-off-by: Jarno Rajahalme --- cilium/bpf_metadata.cc | 34 +-- cilium/grpc_subscription.cc | 36 +-- cilium/grpc_subscription.h | 14 +- cilium/host_map.cc | 6 +- cilium/network_policy.cc | 429 ++++++++++++++++++++-------- cilium/network_policy.h | 179 +----------- cilium/secret_watcher.cc | 43 +-- cilium/secret_watcher.h | 16 +- tests/bpf_metadata.cc | 22 +- tests/bpf_metadata.h | 13 +- tests/cilium_network_policy_test.cc | 11 +- tests/metadata_config_test.cc | 2 +- 12 files changed, 410 insertions(+), 395 deletions(-) diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index fec7b462f..2e787c539 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -176,27 +176,6 @@ SINGLETON_MANAGER_REGISTRATION(cilium_bpf_conntrack); SINGLETON_MANAGER_REGISTRATION(cilium_host_map); SINGLETON_MANAGER_REGISTRATION(cilium_network_policy); -namespace { - -std::shared_ptr -createHostMap(Server::Configuration::ListenerFactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), [&context] { - auto map = std::make_shared(context.serverFactoryContext()); - map->startSubscription(context.serverFactoryContext()); - return map; - }); -} - -std::shared_ptr -createPolicyMap(Server::Configuration::FactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), - [&context] { return std::make_shared(context, true); }); -} - -} // namespace - Config::Config(const ::cilium::BpfMetadata& config, Server::Configuration::ListenerFactoryContext& context) : so_linger_(config.has_original_source_so_linger_time() @@ -236,7 +215,13 @@ Config::Config(const ::cilium::BpfMetadata& config, } if (config.use_nphds()) { - hosts_ = createHostMap(context); + hosts_ = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), [&context] { + auto map = std::make_shared(context.serverFactoryContext()); + map->startSubscription(context.serverFactoryContext()); + return map; + }); } // Note: all instances use the bpf root of the first filter with non-empty @@ -273,7 +258,10 @@ Config::Config(const ::cilium::BpfMetadata& config, // instances! // Only created if either ipcache_ or hosts_ map exists if (ipcache_ || hosts_) { - npmap_ = createPolicyMap(context); + npmap_ = + context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), + [&context] { return std::make_shared(context, true); }); } } diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index b7b1bfd6e..1eb8a6c0b 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -10,14 +10,13 @@ #include "envoy/annotations/resource.pb.h" #include "envoy/common/exception.h" -#include "envoy/common/random_generator.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/custom_config_validators.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/subscription_factory.h" -#include "envoy/event/dispatcher.h" #include "envoy/grpc/async_client.h" -#include "envoy/local_info/local_info.h" +#include "envoy/server/factory_context.h" #include "envoy/stats/scope.h" #include "envoy/upstream/cluster_manager.h" @@ -140,17 +139,16 @@ envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig() { envoy::config::core::v3::ConfigSource cilium_xds_api_config = getCiliumXDSAPIConfig(); -std::unique_ptr -subscribe(const std::string& type_url, const LocalInfo::LocalInfo& local_info, - Upstream::ClusterManager& cm, Event::Dispatcher& dispatcher, - Random::RandomGenerator& random, Stats::Scope& scope, - Config::SubscriptionCallbacks& callbacks, +std::unique_ptr +subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, + Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, std::chrono::milliseconds init_fetch_timeout) { + const envoy::config::core::v3::ConfigSource config_source = getCiliumXDSAPIConfig(); const envoy::config::core::v3::ApiConfigSource& api_config_source = - cilium_xds_api_config.api_config_source(); + config_source.api_config_source(); THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( - cm.primaryClusters(), api_config_source)); + context.clusterManager().primaryClusters(), api_config_source)); Config::SubscriptionStats stats = Config::Utility::generateStats(scope); Envoy::Config::SubscriptionOptions options; @@ -159,7 +157,7 @@ subscribe(const std::string& type_url, const LocalInfo::LocalInfo& local_info, Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = std::make_unique(); auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( - cm.grpcAsyncClientManager(), api_config_source, scope, true, 0, false); + context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, false); THROW_IF_NOT_OK_REF(factory_or_error.status()); absl::StatusOr rate_limit_settings_or_error = @@ -170,9 +168,9 @@ subscribe(const std::string& type_url, const LocalInfo::LocalInfo& local_info, /*async_client_=*/THROW_OR_RETURN_VALUE( factory_or_error.value()->createUncachedRawAsyncClient(), Grpc::RawAsyncClientPtr), /*failover_async_client_=*/nullptr, - /*dispatcher_=*/dispatcher, + /*dispatcher_=*/context.mainThreadDispatcher(), /*service_method_=*/sotwGrpcMethod(type_url), - /*local_info_=*/local_info, + /*local_info_=*/context.localInfo(), /*rate_limit_settings_=*/rate_limit_settings_or_error.value(), /*scope_=*/scope, /*config_validators_=*/std::move(nop_config_validators), @@ -181,16 +179,18 @@ subscribe(const std::string& type_url, const LocalInfo::LocalInfo& local_info, /*backoff_strategy_=*/ std::make_unique( Config::SubscriptionFactory::RetryInitialDelayMs, - Config::SubscriptionFactory::RetryMaxDelayMs, random), + Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), /*target_xds_authority_=*/"", /*eds_resources_cache_=*/nullptr // EDS cache is only used for ADS. }; + std::shared_ptr grpc_mux = + std::static_pointer_cast(std::make_shared( + grpc_mux_context, api_config_source.set_node_on_first_message_only())); + return std::make_unique( - std::make_shared(grpc_mux_context, - api_config_source.set_node_on_first_message_only()), - callbacks, resource_decoder, stats, type_url, dispatcher, init_fetch_timeout, - /*is_aggregated*/ false, options); + grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), + init_fetch_timeout, /*is_aggregated*/ false, options); } } // namespace Cilium diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index 496ee0568..1f830818c 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -4,17 +4,13 @@ #include #include -#include "envoy/common/random_generator.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/local_info/local_info.h" +#include "envoy/server/factory_context.h" #include "envoy/stats/scope.h" -#include "envoy/upstream/cluster_manager.h" #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" #include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" namespace Envoy { namespace Cilium { @@ -46,11 +42,9 @@ class GrpcMuxImpl : public Config::GrpcMuxImpl { bool new_stream_ = true; }; -std::unique_ptr -subscribe(const std::string& type_url, const LocalInfo::LocalInfo& local_info, - Upstream::ClusterManager& cm, Event::Dispatcher& dispatcher, - Random::RandomGenerator& random, Stats::Scope& scope, - Config::SubscriptionCallbacks& callbacks, +std::unique_ptr +subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, + Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); diff --git a/cilium/host_map.cc b/cilium/host_map.cc index 75a35b9d2..d4efaf14e 100644 --- a/cilium/host_map.cc +++ b/cilium/host_map.cc @@ -171,10 +171,8 @@ PolicyHostMap::PolicyHostMap(Server::Configuration::CommonFactoryContext& contex } void PolicyHostMap::startSubscription(Server::Configuration::CommonFactoryContext& context) { - subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicyHosts", context.localInfo(), - context.clusterManager(), context.mainThreadDispatcher(), - context.api().randomGenerator(), *scope_, *this, - std::make_shared()); + subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicyHosts", context, *scope_, + *this, std::make_shared()); subscription_->start({}); } diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index ba9127f8a..c26bed13d 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,9 @@ #include "envoy/http/header_map.h" #include "envoy/init/manager.h" #include "envoy/network/address.h" +#include "envoy/server/config_tracker.h" #include "envoy/server/factory_context.h" +#include "envoy/server/transport_socket_config.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/stats/scope.h" @@ -34,6 +37,7 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" +#include "source/common/common/macros.h" #include "source/common/common/matchers.h" #include "source/common/common/thread.h" #include "source/common/http/header_utility.h" @@ -46,7 +50,9 @@ #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" +#include "absl/container/btree_map.h" #include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" #include "absl/strings/ascii.h" @@ -59,6 +65,19 @@ #include "cilium/ipcache.h" #include "cilium/secret_watcher.h" +namespace Envoy { +namespace Cilium { + +// Supported verdict kinds +using RuleVerdict = enum { + None = 0, + Allow = 1, + Deny = 2, +}; + +} // namespace Cilium +} // namespace Envoy + namespace fmt { template <> struct formatter { @@ -90,6 +109,151 @@ template <> struct formatter { namespace Envoy { namespace Cilium { +class PolicyInstanceImpl; + +using PolicyMapSnapshot = + absl::flat_hash_map>; + +class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, + public Logger::Loggable { +public: + NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context); + ~NetworkPolicyMapImpl() override; + + void startSubscription() { + subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, + *npds_stats_scope_, *this, std::make_shared()); + } + + // This is used for testing with a file-based subscription + void startSubscription(std::unique_ptr&& subscription) { + subscription_ = std::move(subscription); + } + + // Config::SubscriptionCallbacks + absl::Status onConfigUpdate(const std::vector& resources, + const std::string& version_info) override; + absl::Status onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override { + // NOT IMPLEMENTED YET. + UNREFERENCED_PARAMETER(added_resources); + UNREFERENCED_PARAMETER(removed_resources); + UNREFERENCED_PARAMETER(system_version_info); + return absl::OkStatus(); + } + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException* e) override; + + Server::Configuration::TransportSocketFactoryContext& transportFactoryContext() const { + return *transport_factory_context_; + } + + Regex::Engine& regexEngine() const { return context_.regexEngine(); } + + void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } + +protected: + bool isNewStream() const { + auto sub = dynamic_cast(subscription_.get()); + if (!sub) { + ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); + return false; + } + auto mux = dynamic_cast(sub->grpcMux().get()); + if (!mux) { + ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); + return false; + } + return mux->isNewStream(); + } + + // run the given function after all the threads have scheduled + void runAfterAllThreads(std::function cb) const { + // We can guarantee the callback 'cb' runs in the main thread after all worker threads have + // entered their event loop, and thus relinquished all state, such as policy lookup results that + // were stored in their call stack, by posting and empty function to their event queues and + // waiting until all of them have returned, as managed by 'runOnAllWorkerThreads'. + context_.threadLocal().runOnAllWorkerThreads([]() {}, cb); + } + + std::string resourceName(const cilium::NetworkPolicy& config) { + return fmt::format("{}", config.endpoint_id()); + } + + void reopenIpcache(); + + std::shared_ptr + createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); + + void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, + Init::ManagerImpl& version_init_manager, std::string&& version_name); + +private: + // Helpers for atomic swap of the policy map pointer. + // + // store() is only used for the initialization of the map during construction. + // exchange() is used to atomically swap in a new map, the old map pointer is returned. + // Once a map is stored or swapped in to the atomic pointer by the main thread, it may be "loaded" + // from the atomic pointer by any thread. This is why the load returns a const pointer. + // + // For the loaded pointer to be safe to use, we must use acquire/release memory ordering: + // - when a pointer stored or swapped in, 'std::memory_order_release' informs the compiler to make + // sure it is not reordering any write operations into the map to happen after the pointer is + // written, and emits CPU instructions to also make the CPU out-of-order-execution logic to not + // reorder any write operations to happen after the pointer itself is written. This guarantees + // that the map is not modified after the point when the worker threads can observe the new + // pointer value, i.e., the map is actaully immutable (const) from that point forward. + // - when the pointer is read (by a worker thread) 'std::memory_order_acquire' in the load + // operation informs the compiler to emit CPU instructions to make the CPU + // out-of-order-execution logic to not reorder any reads from the new map to happen before the + // pointer itself is read, so that no values from the map are read before the map was "released" + // by the store or exchange operation. + // + // Typically it is easier to think about the release part of the acquire/release semantics, as at + // the point of the store or exchange operation the compiler and the CPU know the location of the + // map in memory before and after the pointer is stored, so that without + // 'std::memory_order_release' there is an understandable risk of such write after release + // happening. On the acquire side it seems less likely that the compiler or the CPU could know the + // new map pointer value in advance and even try to reorder any read operations to happen before + // the pointer is actually read. But consider the typical case where the pointer value is actually + // not changing between consecutice load operations. The compiler or the CPU could speculate that + // to be the case and read some values from the old memory location. 'std::memory_order_acquire' + // tells the compiler (which then "tells" the CPU) that this can not be done, and all reads must + // actually happen after the pointer value is loaded, be it a new one or the same as before. + // + const PolicyMapSnapshot* load() const { return map_ptr_.load(std::memory_order_acquire); } + void store(const PolicyMapSnapshot* map) { map_ptr_.store(map, std::memory_order_release); } + const PolicyMapSnapshot* exchange(const PolicyMapSnapshot* map) { + return map_ptr_.exchange(map, std::memory_order_release); + } + + const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; + void removeInitManager(); + + static uint64_t instance_id_; + + Server::Configuration::ServerFactoryContext& context_; + std::atomic map_ptr_; + Stats::ScopeSharedPtr npds_stats_scope_; + Stats::ScopeSharedPtr policy_stats_scope_; + + // init target which starts gRPC subscription + Init::TargetImpl init_target_; + std::shared_ptr + transport_factory_context_; + + std::unique_ptr subscription_; + + ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; + +protected: + friend class NetworkPolicyMap; + + PolicyStats stats_; +}; + uint64_t NetworkPolicyMapImpl::instance_id_ = 0; IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { @@ -114,7 +278,8 @@ class HeaderMatch : public Logger::Loggable { : name_(config.name()), value_(config.value()), match_action_(config.match_action()), mismatch_action_(config.mismatch_action()) { if (!config.value_sds_secret().empty()) { - secret_ = std::make_unique(parent, config.value_sds_secret()); + secret_ = std::make_unique(parent.transportFactoryContext(), + config.value_sds_secret()); } } @@ -469,11 +634,13 @@ class PortNetworkPolicyRule : public Logger::Loggable { } if (rule.has_downstream_tls_context()) { auto config = rule.downstream_tls_context(); - server_context_ = std::make_unique(parent, config); + server_context_ = + std::make_unique(parent.transportFactoryContext(), config); } if (rule.has_upstream_tls_context()) { auto config = rule.upstream_tls_context(); - client_context_ = std::make_unique(parent, config); + client_context_ = + std::make_unique(parent.transportFactoryContext(), config); } for (const auto& sni : rule.server_names()) { ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): Allowing SNI {} by rule {}", sni, name_); @@ -975,9 +1142,33 @@ class PortNetworkPolicyRules : public Logger::Loggable { bool initialized_{false}; }; +// PortRangeCompare is used for as std::less replacement for port range keys. +// +// All port ranges in the map have non-overlapping keys, which allows total ordering needed for +// ordered map containers. When inserting new ranges, any range overlap will be flagged as a +// "duplicate" entry, as overlapping keys are considered equal (as neither is strictly less than the +// other given this comparison predicate). +// On lookups we'll set both ends of the port range to the same port number, which will find the one +// range that it overlaps with, if one exists. +using PortRange = std::pair; +struct PortRangeCompare { + bool operator()(const PortRange& a, const PortRange& b) const { + // return true if range 'a.first - a.second' is below range 'b.first - b.second'. + return a.second < b.first; + } +}; + +// PolicySnapshot is keyed by port ranges, and contains a list of PortNetworkPolicyRules's +// applicable to this range. A list is needed as rules may come from multiple sources (e.g., +// resulting from use of named ports and numbered ports in Cilium Network Policy at the same time). +class PolicySnapshot : public absl::btree_map { +public: + using absl::btree_map::btree_map; +}; + namespace { -const PortNetworkPolicyRules* findPortRules(const PolicyMap& map, uint16_t port) { +const PortNetworkPolicyRules* findPortRules(const PolicySnapshot& map, uint16_t port) { // Look up with an exact port first, then fall back to the wildcard port (0). If policy is found // with the exact port, then the returned policy also contains all the wildcard port rules, so we // do not need to perform a separate wildcard port policy lookup. If no policy is defined for the @@ -997,8 +1188,8 @@ const PortNetworkPolicyRules* findPortRules(const PolicyMap& map, uint16_t port) } // namespace -PortPolicy::PortPolicy(const PolicyMap& map, uint16_t port) - : map_(map), port_rules_(findPortRules(map_, port)), +PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port) + : port_rules_(findPortRules(map, port)), has_http_rules_(port_rules_ && port_rules_->hasHttpRules()) {} bool PortPolicy::useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { @@ -1752,7 +1943,7 @@ class PortNetworkPolicy : public Logger::Loggable { } } - PolicyMap rules_; + PolicySnapshot rules_; bool has_http_rules_ = false; }; @@ -1825,17 +2016,8 @@ NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& contex : context_(context.serverFactoryContext()) { impl_ = std::make_unique(context); - if (context_.admin().has_value()) { - ENVOY_LOG(debug, "Registering NetworkPolicies to config tracker"); - config_tracker_entry_ = context_.admin()->getConfigTracker().add( - "networkpolicies", [this](const Matchers::StringMatcher& name_matcher) { - return dumpNetworkPolicyConfigs(name_matcher); - }); - RELEASE_ASSERT(config_tracker_entry_, ""); - } - if (subscribe) { - getImpl().startSubscription(); + impl_->startSubscription(); } } @@ -1858,6 +2040,21 @@ NetworkPolicyMap::~NetworkPolicyMap() { context_.mainThreadDispatcher().post([impl = std::move(impl_)]() {}); } +bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceImpl(endpoint_policy_name) != nullptr; +} + +void NetworkPolicyMap::startSubscriptionForTest( + std::unique_ptr&& subscription) { + impl_->startSubscription(std::move(subscription)); +} + +Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksForTest() const { + return *impl_; +} + +PolicyStats& NetworkPolicyMap::statsForTest() const { return impl_->stats_; } + NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context) : context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), @@ -1878,8 +2075,17 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext context.initManager().add(init_target_); // Allocate an initial policy map so that the map pointer is never a nullptr - store(new RawPolicyMap()); + store(new PolicyMapSnapshot()); ENVOY_LOG(trace, "NetworkPolicyMapImpl({}) created.", instance_id_); + + if (context_.admin().has_value()) { + ENVOY_LOG(debug, "Registering NetworkPolicies to config tracker"); + config_tracker_entry_ = context_.admin()->getConfigTracker().add( + "networkpolicies", [this](const Matchers::StringMatcher& name_matcher) { + return dumpNetworkPolicyConfigs(name_matcher); + }); + RELEASE_ASSERT(config_tracker_entry_, ""); + } } // NetworkPolicyMapImpl destructor must only be called from the main thread. @@ -1889,29 +2095,49 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { delete load(); } -void NetworkPolicyMapImpl::startSubscription() { - subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicy", context_.localInfo(), - context_.clusterManager(), context_.mainThreadDispatcher(), - context_.api().randomGenerator(), *npds_stats_scope_, *this, - std::make_shared()); +void NetworkPolicyMapImpl::reopenIpcache() { + // Get ipcache singleton only if it was successfully created previously. + // Cilium agent re-creates IP cache on restart, and the first accepted update on + // the new stream must reopen it before workers enforce refreshed identities. + IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); + if (ipcache != nullptr) { + ENVOY_LOG(info, "Reopening ipcache on new stream"); + ipcache->open(); + } } -void NetworkPolicyMapImpl::tlsWrapperMissingPolicyInc() const { - stats_.tls_wrapper_missing_policy_.inc(); +std::shared_ptr +NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, + const PolicyMapSnapshot& old_policy_map) { + const uint64_t new_hash = MessageUtil::hash(config); + auto policy_it = old_policy_map.find(config.endpoint_ips()[0]); + if (policy_it != old_policy_map.cend()) { + const auto& old_policy = policy_it->second; + if (old_policy && old_policy->hash_ == new_hash && + Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + return old_policy; + } + } + + // May throw + return std::make_shared(*this, new_hash, config); } -bool NetworkPolicyMapImpl::isNewStream() { - auto sub = dynamic_cast(subscription_.get()); - if (!sub) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); - return false; - } - auto mux = dynamic_cast(sub->grpcMux().get()); - if (!mux) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); - return false; - } - return mux->isNewStream(); +void NetworkPolicyMapImpl::installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, + Init::ManagerImpl& version_init_manager, + std::string&& version_name) { + // Initialize SDS secrets. We do not wait for the completion. + version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); + + const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); + + // Delete the old map once all worker threads have entered their event queues, as this + // is proof that they no longer refer to the old map. + runAfterAllThreads([old_policy_map]() { + // Clean-up in the main thread after all threads have scheduled + delete old_policy_map; + }); } // removeInitManager must be called at the end of each policy update @@ -1945,12 +2171,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (isNewStream()) { ENVOY_LOG(info, "New NetworkPolicy stream"); - // Get ipcache singleton only if it was successfully created previously - IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); - if (ipcache != nullptr) { - ENVOY_LOG(info, "Reopening ipcache on new stream"); - ipcache->open(); - } + reopenIpcache(); } std::string version_name = fmt::format("NetworkPolicyMap version {}", version_info); @@ -1960,68 +2181,35 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // SDS secrets will use this! transport_factory_context_->setInitManager(version_init_manager); - const auto* old_map = load(); - RawPolicyMap new_map; - { - try { - for (const auto& resource : resources) { - const auto& config = dynamic_cast(resource.get().resource()); - ENVOY_LOG(debug, - "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " - "version {}", - config.endpoint_id(), config.endpoint_ips()[0], version_info); - if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); - } - - // First find the old config to figure out if an update is needed. - const uint64_t new_hash = MessageUtil::hash(config); - auto it = old_map->find(config.endpoint_ips()[0]); - if (it != old_map->cend()) { - const auto& old_policy = it->second; - if (old_policy && old_policy->hash_ == new_hash && - Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium keeping network policy for endpoint {}", endpoint_ip); - new_map.emplace(endpoint_ip, old_policy); - } - continue; - } - } - - // May throw - auto new_policy = std::make_shared(*this, new_hash, config); - - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); - // new_map is not exception safe, new_policy must be computed separately! - new_map.emplace(endpoint_ip, new_policy); - } + const auto* old_policy_map = load(); + PolicyMapSnapshot new_policy_map; + try { + for (const auto& resource : resources) { + const auto& config = dynamic_cast(resource.get().resource()); + if (config.endpoint_ips().empty()) { + throw EnvoyException("Network Policy has no endpoint ips"); + } + ENVOY_LOG(debug, + "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " + "version {}", + config.endpoint_id(), config.endpoint_ips()[0], version_info); + + auto policy = createOrReusePolicy(config, *old_policy_map); + for (const auto& endpoint_ip : config.endpoint_ips()) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", endpoint_ip); + // new_policy_map is not exception safe, policy must be computed separately! + new_policy_map.emplace(endpoint_ip, policy); } - } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); - stats_.updates_rejected_.inc(); - - removeInitManager(); - throw; // re-throw } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); + stats_.updates_rejected_.inc(); removeInitManager(); - - // Initialize SDS secrets. We do not wait for the completion. - version_init_manager.initialize(Init::WatcherImpl(version_name, []() {})); - - // Swap the new map in, new_map goes out of scope right after to eliminate accidental - // modification. - old_map = exchange(new RawPolicyMap(std::move(new_map))); + throw; // re-throw } + removeInitManager(); - // Delete the old map once all worker threads have entered their event queues, as this - // is proof that they no longer refer to the old map. - runAfterAllThreads([old_map]() { - // Clean-up in the main thread after all threads have scheduled - delete old_map; - }); + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name)); return absl::OkStatus(); } @@ -2033,26 +2221,13 @@ void NetworkPolicyMapImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailu ENVOY_LOG(debug, "Network Policy Update failed, keeping existing policy."); } -void NetworkPolicyMapImpl::runAfterAllThreads(std::function cb) const { - // We can guarantee the callback 'cb' runs in the main thread after all worker threads have - // entered their event loop, and thus relinquished all state, such as policy lookup results that - // were stored in their call stack, by posting and empty function to their event queues and - // waiting until all of them have returned, as managed by 'runOnAllWorkerThreads'. - // - // For now we rely on the implementation dependent fact that the reference returned by - // context_.threadLocal() actually is a ThreadLocal::Instance reference, where - // runOnAllWorkerThreads() is exposed. Without this cast we'd need to use a dummy thread local - // variable that would take a thread local slot for no other purpose than to avoid this type cast. - dynamic_cast(context_.threadLocal()).runOnAllWorkerThreads([]() {}, cb); -} - ProtobufTypes::MessagePtr -NetworkPolicyMap::dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher) { +NetworkPolicyMapImpl::dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher) { ENVOY_LOG(debug, "Writing NetworkPolicies to NetworkPoliciesConfigDump"); std::vector policy_endpoint_ids; auto config_dump = std::make_unique(); - for (const auto& item : *getImpl().load()) { + for (const auto& item : *load()) { // filter duplicates (policies are stored per endpoint ip) if (std::find(policy_endpoint_ids.begin(), policy_endpoint_ids.end(), item.second->policy_proto_.endpoint_id()) != policy_endpoint_ids.end()) { @@ -2103,16 +2278,17 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { void tlsWrapperMissingPolicyInc() const override {} private: - PolicyMap empty_map_; + PolicySnapshot empty_map_; static const std::string empty_string; static const IpAddressPair empty_ips; }; const std::string AllowAllEgressPolicyInstanceImpl::empty_string = ""; const IpAddressPair AllowAllEgressPolicyInstanceImpl::empty_ips{}; -AllowAllEgressPolicyInstanceImpl NetworkPolicyMap::AllowAllEgressPolicy; - -PolicyInstance& NetworkPolicyMap::getAllowAllEgressPolicy() { return AllowAllEgressPolicy; } +PolicyInstance& NetworkPolicyMap::getAllowAllEgressPolicy() { + static AllowAllEgressPolicyInstanceImpl allow_all_egress_policy; + return allow_all_egress_policy; +} // Deny-all policy class DenyAllPolicyInstanceImpl : public PolicyInstance { @@ -2145,16 +2321,17 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { void tlsWrapperMissingPolicyInc() const override {} private: - PolicyMap empty_map_; + PolicySnapshot empty_map_; static const std::string empty_string; static const IpAddressPair empty_ips; }; const std::string DenyAllPolicyInstanceImpl::empty_string = ""; const IpAddressPair DenyAllPolicyInstanceImpl::empty_ips{}; -DenyAllPolicyInstanceImpl NetworkPolicyMap::DenyAllPolicy; - -PolicyInstance& NetworkPolicyMap::getDenyAllPolicy() { return DenyAllPolicy; } +PolicyInstance& NetworkPolicyMap::getDenyAllPolicy() { + static DenyAllPolicyInstanceImpl deny_all_policy; + return deny_all_policy; +} const PolicyInstance* NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) const { @@ -2179,10 +2356,10 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons // for Cilium Ingress when there is no egress policy enforcement for the Ingress traffic. const PolicyInstance& NetworkPolicyMap::getPolicyInstance(const std::string& endpoint_ip, bool default_allow_egress) const { - const auto* policy = getImpl().getPolicyInstanceImpl(endpoint_ip); + const auto* policy = impl_->getPolicyInstanceImpl(endpoint_ip); return policy != nullptr ? *policy - : default_allow_egress ? *static_cast(&AllowAllEgressPolicy) - : *static_cast(&DenyAllPolicy); + : default_allow_egress ? getAllowAllEgressPolicy() + : getDenyAllPolicy(); } } // namespace Cilium diff --git a/cilium/network_policy.h b/cilium/network_policy.h index 1d8a2062c..69d7c7558 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -2,16 +2,10 @@ #include -#include #include -#include #include #include -#include -#include -#include "envoy/common/exception.h" -#include "envoy/common/matchers.h" #include "envoy/common/pure.h" #include "envoy/common/regex.h" #include "envoy/config/core/v3/base.pb.h" @@ -19,28 +13,20 @@ #include "envoy/http/header_map.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" -#include "envoy/server/config_tracker.h" #include "envoy/server/factory_context.h" -#include "envoy/server/transport_socket_config.h" #include "envoy/singleton/instance.h" #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" -#include "envoy/stats/scope.h" #include "envoy/stats/stats_macros.h" // IWYU pragma: keep #include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/macros.h" #include "source/common/common/thread.h" -#include "source/common/init/target_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/server/transport_socket_config_impl.h" -#include "absl/container/btree_map.h" -#include "absl/container/flat_hash_map.h" -#include "absl/status/status.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" @@ -51,35 +37,8 @@ namespace Envoy { namespace Cilium { -// PortRangeCompare is used for as std::less replacement for port range keys. -// -// All port ranges in the map have non-overlapping keys, which allows total ordering needed for -// ordered map containers. When inserting new ranges, any range overlap will be flagged as a -// "duplicate" entry, as overlapping keys are considered equal (as neither is strictly less than the -// other given this comparison predicate). -// On lookups we'll set both ends of the port range to the same port number, which will find the one -// range that it overlaps with, if one exists. -using PortRange = std::pair; -struct PortRangeCompare { - bool operator()(const PortRange& a, const PortRange& b) const { - // return true if range 'a.first - a.second' is below range 'b.first - b.second'. - return a.second < b.first; - } -}; - class PortNetworkPolicyRules; - -// PolicyMap is keyed by port ranges, and contains a list of PortNetworkPolicyRules's applicable -// to this range. A list is needed as rules may come from multiple sources (e.g., resulting from -// use of named ports and numbered ports in Cilium Network Policy at the same time). -using PolicyMap = absl::btree_map; - -// Supported message types -using RuleVerdict = enum { - None = 0, - Allow = 1, - Deny = 2, -}; +class PolicySnapshot; // PortPolicy holds a reference to a set of rules in a policy map that apply to the given port. // Methods then iterate through the set to determine if policy allows or denies. This is needed to @@ -90,7 +49,7 @@ class PortPolicy : public Logger::Loggable { friend class PortNetworkPolicy; friend class DenyAllPolicyInstanceImpl; friend class AllowAllEgressPolicyInstanceImpl; - PortPolicy(const PolicyMap& map, uint16_t port); + PortPolicy(const PolicySnapshot& map, uint16_t port); public: // If hasHttpRules() returns false, then HTTP policy enforcement can be skipped, @@ -126,7 +85,6 @@ class PortPolicy : public Logger::Loggable { bool& raw_socket_allowed) const; private: - const PolicyMap& map_; // using raw pointers by design: // - pointer to distinguish between no rules and empty rules // - not using shared pointer to not allow a worker thread to hold the last reference to policy @@ -184,8 +142,6 @@ class PolicyInstance { }; using PolicyInstanceConstSharedPtr = std::shared_ptr; -class PolicyInstanceImpl; - class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { public: NetworkPolicyDecoder() : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} @@ -227,146 +183,31 @@ struct PolicyStats { ALL_CILIUM_POLICY_STATS(GENERATE_COUNTER_STRUCT, GENERATE_HISTOGRAM_STRUCT) }; -using RawPolicyMap = absl::flat_hash_map>; - -class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, - public Logger::Loggable { -public: - NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context); - ~NetworkPolicyMapImpl() override; - - void startSubscription(); - - // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { - subscription_ = std::move(subscription); - } - - // run the given function after all the threads have scheduled - void runAfterAllThreads(std::function) const; - - // Config::SubscriptionCallbacks - absl::Status onConfigUpdate(const std::vector& resources, - const std::string& version_info) override; - absl::Status onConfigUpdate(const std::vector& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } - void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, - const EnvoyException* e) override; - - Server::Configuration::TransportSocketFactoryContext& transportFactoryContext() const { - return *transport_factory_context_; - } - - Regex::Engine& regexEngine() const { return context_.regexEngine(); } - - void tlsWrapperMissingPolicyInc() const; - -private: - // Helpers for atomic swap of the policy map pointer. - // - // store() is only used for the initialization of the map during construction. - // exchange() is used to atomically swap in a new map, the old map pointer is returned. - // Once a map is stored or swapped in to the atomic pointer by the main thread, it may be "loaded" - // from the atomic pointer by any thread. This is why the load returns a const pointer. - // - // For the loaded pointer to be safe to use, we must use acquire/release memory ordering: - // - when a pointer stored or swapped in, 'std::memory_order_release' informs the compiler to make - // sure it is not reordering any write operations into the map to happen after the pointer is - // written, and emits CPU instructions to also make the CPU out-of-order-execution logic to not - // reorder any write operations to happen after the pointer itself is written. This guarantees - // that the map is not modified after the point when the worker threads can observe the new - // pointer value, i.e., the map is actaully immutable (const) from that point forward. - // - when the pointer is read (by a worker thread) 'std::memory_order_acquire' in the load - // operation informs the compiler to emit CPU instructions to make the CPU - // out-of-order-execution logic to not reorder any reads from the new map to happen before the - // pointer itself is read, so that no values from the map are read before the map was "released" - // by the store or exchange operation. - // - // Typically it is easier to think about the release part of the acquire/release semantics, as at - // the point of the store or exchange operation the compiler and the CPU know the location of the - // map in memory before and after the pointer is stored, so that without - // 'std::memory_order_release' there is an understandable risk of such write after release - // happening. On the acquire side it seems less likely that the compiler or the CPU could know the - // new map pointer value in advance and even try to reorder any read operations to happen before - // the pointer is actually read. But consider the typical case where the pointer value is actually - // not changing between consecutice load operations. The compiler or the CPU could speculate that - // to be the case and read some values from the old memory location. 'std::memory_order_acquire' - // tells the compiler (which then "tells" the CPU) that this can not be done, and all reads must - // actually happen after the pointer value is loaded, be it a new one or the same as before. - // - const RawPolicyMap* load() const { return map_ptr_.load(std::memory_order_acquire); } - void store(const RawPolicyMap* map) { map_ptr_.store(map, std::memory_order_release); } - const RawPolicyMap* exchange(const RawPolicyMap* map) { - return map_ptr_.exchange(map, std::memory_order_release); - } - - const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; - - void removeInitManager(); - - bool isNewStream(); - - static uint64_t instance_id_; - - Server::Configuration::ServerFactoryContext& context_; - std::atomic map_ptr_; - Stats::ScopeSharedPtr npds_stats_scope_; - Stats::ScopeSharedPtr policy_stats_scope_; - - // init target which starts gRPC subscription - Init::TargetImpl init_target_; - std::shared_ptr - transport_factory_context_; - - std::unique_ptr subscription_; - -protected: - friend class NetworkPolicyMap; - friend class CiliumNetworkPolicyTest; - - PolicyStats stats_; -}; - -class DenyAllPolicyInstanceImpl; -class AllowAllEgressPolicyInstanceImpl; +class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe = false); ~NetworkPolicyMap() override; - // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { - getImpl().startSubscription(std::move(subscription)); - } + bool exists(const std::string& endpoint_policy_name) const; const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; - static DenyAllPolicyInstanceImpl DenyAllPolicy; static PolicyInstance& getDenyAllPolicy(); - static AllowAllEgressPolicyInstanceImpl AllowAllEgressPolicy; static PolicyInstance& getAllowAllEgressPolicy(); - bool exists(const std::string& endpoint_policy_name) const { - return getImpl().getPolicyInstanceImpl(endpoint_policy_name) != nullptr; - } - - NetworkPolicyMapImpl& getImpl() const { return *impl_; } +protected: + friend class CiliumNetworkPolicyTest; + friend struct TestHelper; + PolicyStats& statsForTest() const; + void startSubscriptionForTest(std::unique_ptr&& subscription); + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: Server::Configuration::ServerFactoryContext& context_; std::unique_ptr impl_; - - ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); - Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; }; using NetworkPolicyMapSharedPtr = std::shared_ptr; diff --git a/cilium/secret_watcher.cc b/cilium/secret_watcher.cc index 2155b776d..387431396 100644 --- a/cilium/secret_watcher.cc +++ b/cilium/secret_watcher.cc @@ -24,7 +24,6 @@ #include "absl/synchronization/mutex.h" #include "cilium/api/npds.pb.h" #include "cilium/grpc_subscription.h" -#include "cilium/network_policy.h" namespace Envoy { namespace Cilium { @@ -51,9 +50,9 @@ GetSdsConfigFunc getSDSConfig = &getCiliumSDSConfig; void setSDSConfigFunc(GetSdsConfigFunc func) { getSDSConfig = func; } void resetSDSConfigFunc() { getSDSConfig = &getCiliumSDSConfig; } -SecretWatcher::SecretWatcher(const NetworkPolicyMapImpl& parent, const std::string& sds_name) - : parent_(parent), name_(sds_name), - secret_provider_(secretProvider(parent.transportFactoryContext(), sds_name)), +SecretWatcher::SecretWatcher(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& sds_name) + : context_(context), name_(sds_name), secret_provider_(secretProvider(context, sds_name)), update_secret_(readAndWatchSecret()) {} SecretWatcher::~SecretWatcher() { @@ -72,7 +71,7 @@ Envoy::Common::CallbackHandlePtr SecretWatcher::readAndWatchSecret() { absl::Status SecretWatcher::store() { const auto* secret = secret_provider_->secret(); if (secret != nullptr) { - Api::Api& api = parent_.transportFactoryContext().serverFactoryContext().api(); + Api::Api& api = context_.serverFactoryContext().api(); auto string_or_error = Config::DataSource::read(secret->secret(), true, api); if (!string_or_error.ok()) { return string_or_error.status(); @@ -80,8 +79,9 @@ absl::Status SecretWatcher::store() { std::string* p = new std::string(string_or_error.value()); std::string* old = ptr_.exchange(p, std::memory_order_release); if (old != nullptr) { - // Delete old value after all threads have scheduled - parent_.runAfterAllThreads([old]() { delete old; }); + // Delete old value after all worker threads have scheduled + context_.serverFactoryContext().threadLocal().runOnAllWorkerThreads([]() {}, + [old]() { delete old; }); } } return absl::OkStatus(); @@ -89,9 +89,10 @@ absl::Status SecretWatcher::store() { const std::string* SecretWatcher::load() const { return ptr_.load(std::memory_order_acquire); } -TLSContext::TLSContext(const NetworkPolicyMapImpl& parent, const std::string& name) - : manager_(parent.transportFactoryContext().serverFactoryContext().sslContextManager()), - scope_(parent.transportFactoryContext().serverFactoryContext().serverScope()), +TLSContext::TLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& name) + : manager_(context.serverFactoryContext().sslContextManager()), + scope_(context.serverFactoryContext().serverScope()), init_target_(fmt::format("TLS Context {} secret", name), []() {}) {} namespace { @@ -134,9 +135,9 @@ void setCommonConfig(const cilium::TLSContext config, } // namespace -DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, - const cilium::TLSContext config) - : TLSContext(parent, "server") { +DownstreamTLSContext::DownstreamTLSContext( + Server::Configuration::TransportSocketFactoryContext& context, const cilium::TLSContext config) + : TLSContext(context, "server") { // Server config always needs the TLS certificate to present to the client if (config.tls_sds_secret().empty() && config.certificate_chain().empty()) { throw EnvoyException("Downstream TLS Context: missing certificate chain"); @@ -156,7 +157,7 @@ DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, server_names_.emplace_back(config.server_names(i)); } auto server_config_or_error = Extensions::TransportSockets::Tls::ServerContextConfigImpl::create( - context_config, parent.transportFactoryContext(), false); + context_config, context, false); // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) THROW_IF_NOT_OK(server_config_or_error.status()); server_config_ = std::move(server_config_or_error.value()); @@ -180,13 +181,13 @@ DownstreamTLSContext::DownstreamTLSContext(const NetworkPolicyMapImpl& parent, if (server_config_->isReady()) { static_cast(create_server_context()); } else { - parent.transportFactoryContext().initManager().add(init_target_); + context.initManager().add(init_target_); } } -UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, - cilium::TLSContext config) - : TLSContext(parent, "client") { +UpstreamTLSContext::UpstreamTLSContext( + Server::Configuration::TransportSocketFactoryContext& context, cilium::TLSContext config) + : TLSContext(context, "client") { // Client context always needs the trusted CA for server certificate validation // TODO: Default to system default trusted CAs? if (config.validation_context_sds_secret().empty() && config.trusted_ca().empty()) { @@ -203,8 +204,8 @@ UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, } context_config.set_sni(config.server_names(0)); } - auto client_config_or_error = Extensions::TransportSockets::Tls::ClientContextConfigImpl::create( - context_config, parent.transportFactoryContext()); + auto client_config_or_error = + Extensions::TransportSockets::Tls::ClientContextConfigImpl::create(context_config, context); // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) THROW_IF_NOT_OK(client_config_or_error.status()); @@ -227,7 +228,7 @@ UpstreamTLSContext::UpstreamTLSContext(const NetworkPolicyMapImpl& parent, if (client_config_->isReady()) { static_cast(create_client_context()); } else { - parent.transportFactoryContext().initManager().add(init_target_); + context.initManager().add(init_target_); } } diff --git a/cilium/secret_watcher.h b/cilium/secret_watcher.h index 18992f0b8..024e14773 100644 --- a/cilium/secret_watcher.h +++ b/cilium/secret_watcher.h @@ -11,6 +11,7 @@ #include "envoy/ssl/context.h" #include "envoy/ssl/context_config.h" #include "envoy/ssl/context_manager.h" +#include "envoy/ssl/private_key/private_key.h" #include "envoy/stats/scope.h" #include "source/common/common/logger.h" @@ -20,7 +21,6 @@ #include "absl/status/status.h" #include "absl/synchronization/mutex.h" #include "cilium/api/npds.pb.h" -#include "cilium/network_policy.h" namespace Envoy { namespace Cilium { @@ -33,7 +33,8 @@ void resetSDSConfigFunc(); class SecretWatcher : public Logger::Loggable { public: - SecretWatcher(const NetworkPolicyMapImpl& parent, const std::string& sds_name); + SecretWatcher(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& sds_name); ~SecretWatcher(); const std::string& name() const { return name_; } @@ -44,7 +45,7 @@ class SecretWatcher : public Logger::Loggable { absl::Status store(); const std::string* load() const; - const NetworkPolicyMapImpl& parent_; + Server::Configuration::TransportSocketFactoryContext& context_; const std::string name_; std::atomic ptr_{nullptr}; Secret::GenericSecretConfigProviderSharedPtr secret_provider_; @@ -58,7 +59,8 @@ class TLSContext : public Logger::Loggable { TLSContext() = delete; protected: - TLSContext(const NetworkPolicyMapImpl& parent, const std::string& name); + TLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const std::string& name); Envoy::Ssl::ContextManager& manager_; Stats::Scope& scope_; @@ -68,7 +70,8 @@ class TLSContext : public Logger::Loggable { class DownstreamTLSContext : protected TLSContext { public: - DownstreamTLSContext(const NetworkPolicyMapImpl& parent, const cilium::TLSContext config); + DownstreamTLSContext(Server::Configuration::TransportSocketFactoryContext& context, + const cilium::TLSContext config); ~DownstreamTLSContext() { manager_.removeContext(server_context_); } const Ssl::ContextConfig& getTlsContextConfig() const { return *server_config_; } @@ -87,7 +90,8 @@ using DownstreamTLSContextSharedPtr = std::shared_ptr; class UpstreamTLSContext : protected TLSContext { public: - UpstreamTLSContext(const NetworkPolicyMapImpl& parent, cilium::TLSContext config); + UpstreamTLSContext(Server::Configuration::TransportSocketFactoryContext& context, + cilium::TLSContext config); ~UpstreamTLSContext() { manager_.removeContext(client_context_); } const Ssl::ContextConfig& getTlsContextConfig() const { return *client_config_; } diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index 828265858..f2f56aea1 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -49,10 +49,11 @@ std::string policy_path = ""; std::vector> sds_configs{}; -namespace { +namespace Cilium { std::shared_ptr -createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext& context) { +TestHelper::createHostMap(const std::string& config, + Server::Configuration::ListenerFactoryContext& context) { return context.serverFactoryContext().singletonManager().getTyped( "cilium_host_map_singleton", [&config, &context] { std::string path = TestEnvironment::writeStringToFileForTest("host_map.yaml", config); @@ -75,9 +76,9 @@ createHostMap(const std::string& config, Server::Configuration::ListenerFactoryC } std::shared_ptr -createPolicyMap(const std::string& config, - const std::vector>& secret_configs, - Server::Configuration::FactoryContext& context) { +TestHelper::createPolicyMap(const std::string& config, + const std::vector>& secret_configs, + Server::Configuration::FactoryContext& context) { return context.serverFactoryContext().singletonManager().getTyped( "cilium_network_policy_singleton", [&config, &secret_configs, &context] { if (!secret_configs.empty()) { @@ -114,17 +115,15 @@ createPolicyMap(const std::string& config, auto map = std::make_shared(context); auto subscription = std::make_unique( context.serverFactoryContext().mainThreadDispatcher(), - Envoy::Config::makePathConfigSource(policy_path), map->getImpl(), + Envoy::Config::makePathConfigSource(policy_path), map->subscriptionCallbacksForTest(), std::make_shared(), stats, ProtobufMessage::getNullValidationVisitor(), context.serverFactoryContext().api()); - map->startSubscription(std::move(subscription)); + map->startSubscriptionForTest(std::move(subscription)); return map; }); } -} // namespace - -void initTestMaps(Server::Configuration::ListenerFactoryContext& context) { +void TestHelper::initTestMaps(Server::Configuration::ListenerFactoryContext& context) { // Create the file-based policy map before the filter is created, so that the // singleton is set before the gRPC subscription is attempted. hostmap = createHostMap(host_map_config, context); @@ -133,7 +132,6 @@ void initTestMaps(Server::Configuration::ListenerFactoryContext& context) { npmap = createPolicyMap(policy_config, sds_configs, context); } -namespace Cilium { namespace BpfMetadata { namespace { @@ -209,7 +207,7 @@ class TestBpfMetadataConfigFactory : public NamedListenerFilterConfigFactory { const Network::ListenerFilterMatcherSharedPtr& listener_filter_matcher, ListenerFactoryContext& context) override { - initTestMaps(context); + Cilium::TestHelper::initTestMaps(context); auto config = std::make_shared( MessageUtil::downcastAndValidate( diff --git a/tests/bpf_metadata.h b/tests/bpf_metadata.h index e57b29f09..ecdf80e22 100644 --- a/tests/bpf_metadata.h +++ b/tests/bpf_metadata.h @@ -27,9 +27,18 @@ extern std::string policy_config; extern std::string policy_path; extern std::vector> sds_configs; -extern void initTestMaps(Server::Configuration::ListenerFactoryContext& context); - namespace Cilium { + +struct TestHelper { + static std::shared_ptr + createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext&); + static std::shared_ptr + createPolicyMap(const std::string& config, + const std::vector>& secret_configs, + Server::Configuration::FactoryContext&); + static void initTestMaps(Server::Configuration::ListenerFactoryContext&); +}; + namespace BpfMetadata { class TestConfig : public Config { diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index d597ae8bb..d1631998f 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -10,6 +10,7 @@ #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/subscription.h" #include "envoy/init/manager.h" #include "envoy/server/factory_context.h" #include "envoy/service/discovery/v3/discovery.pb.h" @@ -79,6 +80,10 @@ class CiliumNetworkPolicyTest : public ::testing::Test { policy_map_.reset(); } + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacks() const { + return policy_map_->subscriptionCallbacksForTest(); + } + std::string updateFromYaml(const std::string& config) { envoy::service::discovery::v3::DiscoveryResponse message; MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); @@ -88,7 +93,7 @@ class CiliumNetworkPolicyTest : public ::testing::Test { THROW_IF_NOT_OK_REF(decoded_resources_or_error.status()); const auto decoded_resources = std::move(decoded_resources_or_error.value().get()); - EXPECT_TRUE(policy_map_->getImpl() + EXPECT_TRUE(subscriptionCallbacks() .onConfigUpdate(decoded_resources->refvec_, message.version_info()) .ok()); return message.version_info(); @@ -210,7 +215,7 @@ class CiliumNetworkPolicyTest : public ::testing::Test { } std::string updatesRejectedStatName() { - return policy_map_->getImpl().stats_.updates_rejected_.name(); + return policy_map_->statsForTest().updates_rejected_.name(); } NiceMock factory_context_; @@ -225,7 +230,7 @@ TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { } TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) { - EXPECT_TRUE(policy_map_->getImpl().onConfigUpdate({}, "1").ok()); + EXPECT_TRUE(subscriptionCallbacks().onConfigUpdate({}, "1").ok()); EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } diff --git a/tests/metadata_config_test.cc b/tests/metadata_config_test.cc index d2b5c668c..21afbfcd3 100644 --- a/tests/metadata_config_test.cc +++ b/tests/metadata_config_test.cc @@ -197,7 +197,7 @@ class MetadataConfigTest : public testing::Test { host_map_config += extra_host_map_config; policy_config += extra_policy_config; - initTestMaps(context_); + Cilium::TestHelper::initTestMaps(context_); Init::WatcherImpl watcher("metadata test", []() {}); context_.initManager().initialize(watcher); From bccefe6b77cd4fc451fa7f712dfb02be7aca0c41 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 2 Apr 2026 17:02:22 +0200 Subject: [PATCH 04/14] policy: detect new streams for delta and file-based substriptions Signed-off-by: Jarno Rajahalme --- cilium/grpc_subscription.cc | 63 ++++++++++++++++++++++++++++++++++++- cilium/grpc_subscription.h | 34 +++++--------------- cilium/network_policy.cc | 38 +++++++++++----------- 3 files changed, 89 insertions(+), 46 deletions(-) diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index 1eb8a6c0b..bcd11db0c 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -26,7 +27,9 @@ #include "source/common/grpc/common.h" #include "source/common/protobuf/protobuf.h" // IWYU pragma: keep #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "absl/container/flat_hash_map.h" #include "absl/status/statusor.h" @@ -39,6 +42,50 @@ namespace Cilium { namespace { +constexpr uint64_t FirstStreamGeneration = 1; + +class StreamTrackedGrpcMux { +public: + virtual ~StreamTrackedGrpcMux() = default; + virtual uint64_t streamGeneration() const = 0; +}; + +class SotwGrpcMuxImpl : public Config::GrpcMuxImpl, public StreamTrackedGrpcMux { +public: + SotwGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) + : Config::GrpcMuxImpl(grpc_mux_context, skip_subsequent_node) {} + + ~SotwGrpcMuxImpl() override = default; + + void onStreamEstablished() override { + ++stream_generation_; + Config::GrpcMuxImpl::onStreamEstablished(); + } + + uint64_t streamGeneration() const override { return stream_generation_; } + +private: + uint64_t stream_generation_{0}; +}; + +class DeltaGrpcMuxImpl : public Config::NewGrpcMuxImpl, public StreamTrackedGrpcMux { +public: + explicit DeltaGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context) + : Config::NewGrpcMuxImpl(grpc_mux_context) {} + + ~DeltaGrpcMuxImpl() override = default; + + void onStreamEstablished() override { + ++stream_generation_; + Config::NewGrpcMuxImpl::onStreamEstablished(); + } + + uint64_t streamGeneration() const override { return stream_generation_; } + +private: + uint64_t stream_generation_{0}; +}; + // service RPC method fully qualified names. struct Service { std::string sotw_grpc_method_; @@ -139,6 +186,20 @@ envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig() { envoy::config::core::v3::ConfigSource cilium_xds_api_config = getCiliumXDSAPIConfig(); +uint64_t grpcStreamGeneration(Config::Subscription* subscription) { + auto* sub = dynamic_cast(subscription); + if (!sub) { + return FirstStreamGeneration; + } + + auto* grpc_mux = dynamic_cast(sub->grpcMux().get()); + if (grpc_mux == nullptr) { + return FirstStreamGeneration; + } + + return grpc_mux->streamGeneration(); +} + std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, @@ -185,7 +246,7 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte }; std::shared_ptr grpc_mux = - std::static_pointer_cast(std::make_shared( + std::static_pointer_cast(std::make_shared( grpc_mux_context, api_config_source.set_node_on_first_message_only())); return std::make_unique( diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index 1f830818c..bffa2359e 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -9,44 +10,23 @@ #include "envoy/server/factory_context.h" #include "envoy/stats/scope.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" - namespace Envoy { namespace Cilium { // Cilium XDS API config source. Used for all Cilium XDS. extern envoy::config::core::v3::ConfigSource cilium_xds_api_config; -// GrpcMux wrapper to get access to control plane identifier -class GrpcMuxImpl : public Config::GrpcMuxImpl { -public: - GrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) - : Config::GrpcMuxImpl(grpc_mux_context, skip_subsequent_node) {} - - ~GrpcMuxImpl() override = default; - - void onStreamEstablished() override { - new_stream_ = true; - Config::GrpcMuxImpl::onStreamEstablished(); - } - - // isNewStream returns true for the first call after a new stream has been established - bool isNewStream() { - bool new_stream = new_stream_; - new_stream_ = false; - return new_stream; - } - -private: - bool new_stream_ = true; -}; - std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); +// Returns a monotonic stream generation for Cilium subscriptions. +// Value 0 is reserved for policy-map detection of the initial stream and may be returned for +// tracked gRPC subscriptions before any stream has been established. +// Non-gRPC subscriptions and subscriptions without stream tracking are treated as generation 1. +uint64_t grpcStreamGeneration(Config::Subscription* subscription); + } // namespace Cilium } // namespace Envoy diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index c26bed13d..0e876f224 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -47,7 +47,6 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" #include "absl/container/btree_map.h" @@ -154,19 +153,9 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } protected: - bool isNewStream() const { - auto sub = dynamic_cast(subscription_.get()); - if (!sub) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); - return false; - } - auto mux = dynamic_cast(sub->grpcMux().get()); - if (!mux) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); - return false; - } - return mux->isNewStream(); - } + uint64_t streamGeneration() const { return grpcStreamGeneration(subscription_.get()); } + + void resetStreamForTest() { applied_stream_generation_ = 0; } // run the given function after all the threads have scheduled void runAfterAllThreads(std::function cb) const { @@ -187,7 +176,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, std::string&& version_name); + Init::ManagerImpl& version_init_manager, std::string&& version_name, + uint64_t stream_generation); private: // Helpers for atomic swap of the policy map pointer. @@ -244,6 +234,10 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, transport_factory_context_; std::unique_ptr subscription_; + // Value 0 is reserved for detection of the initial stream before the first + // successful policy install. Tracked gRPC subscriptions may also report 0 + // before any stream has been established. + uint64_t applied_stream_generation_{0}; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -2126,12 +2120,17 @@ NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, void NetworkPolicyMapImpl::installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, - std::string&& version_name) { + std::string&& version_name, + uint64_t stream_generation) { // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); + // Record stream state only after a successful install. The reserved value 0 + // keeps the initial accepted update on any stream source classified as new. + applied_stream_generation_ = stream_generation; + // Delete the old map once all worker threads have entered their event queues, as this // is proof that they no longer refer to the old map. runAfterAllThreads([old_policy_map]() { @@ -2160,6 +2159,8 @@ void NetworkPolicyMapImpl::removeInitManager() { absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { + auto stream_generation = streamGeneration(); + const bool is_new_stream = stream_generation != applied_stream_generation_; ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", instance_id_, resources.size(), version_info); stats_.updates_total_.inc(); @@ -2168,7 +2169,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // and that is also when the old stream terminates and a new one is created. // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. - if (isNewStream()) { + if (is_new_stream) { ENVOY_LOG(info, "New NetworkPolicy stream"); reopenIpcache(); @@ -2209,7 +2210,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( } removeInitManager(); - installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name)); + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), + stream_generation); return absl::OkStatus(); } From c63b35a387647971a3abaa3a9ebe58a03ea09039 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sun, 5 Apr 2026 08:10:06 +0200 Subject: [PATCH 05/14] policy: Fix pass precedence test Leave enough space after the pass verdict for all the passed rules to fit in before the following rules on the same tier. This is the requirement of the current API for correct behavior. Signed-off-by: Jarno Rajahalme --- tests/cilium_network_policy_test.cc | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index d1631998f..f05ee3993 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -1907,7 +1907,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { remote_policies: [ 42 ] - port: 80 rules: - - precedence: 850 + - precedence: 750 deny: true - precedence: 600 remote_policies: [ 41, 42, 43 ] @@ -1926,10 +1926,16 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { - rules: - remotes: [41] deny: true - precedence: 1150 + precedence: 1050 + - remotes: [42] + precedence: 800 + http_rules: + - headers: + - name: ":path" + value: "/multi-tier" - remotes: [] deny: true - precedence: 850 + precedence: 750 egress: rules: [] )EOF"; @@ -1938,8 +1944,8 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { // Remote 41 hits the promoted deny from tier 1. EXPECT_FALSE(ingressAllowed("10.1.2.3", 41, 80, {{":path", "/multi-tier"}})); - // Remote 42 is promoted by the lower wildcard tier, but remains below deny. - EXPECT_FALSE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); + // Remote 42 is promoted by the lower wildcard tier + EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); // Remote 43 is not promoted and is denied. EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/multi-tier"}})); @@ -1977,10 +1983,10 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EnvoyException, "PortNetworkPolicy: Inconsistent pass precedence 600 != 700"); - // Failed update must leave policy unchanged from version 10. + // Failed update must leave policy unchanged from version 14. EXPECT_TRUE(validate("10.1.2.3", expected14)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 41, 80, {{":path", "/multi-tier"}})); - EXPECT_FALSE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); // // 16th update: inherited wildcard pass skips remaining rules on that tier From 83ea9010b242cb3739b927da3d1c6060d387b761 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sat, 4 Apr 2026 21:07:23 +0200 Subject: [PATCH 06/14] policy: Handle pass rules during enforcement Handle pass rules during enforcement by skipping past the passed-over rules when a rule with a pass verdict matches. This is more straightforward and gets rid of precedence promotioin altogether, which is made possible by the duplication of wildcard-port rules into the port-specific rulesets, as after that change there is no need to compare precedences between multiple lookups. Since there is no transformation on the rules, the applied rules have the same shape as the imported policy, which makes debugging easier. The implementation skips over the passed-over rules without scanning all of them so the performance should be similar to the previous implementation. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 621 ++++++---------------------- tests/cilium_network_policy_test.cc | 212 ++++++---- 2 files changed, 249 insertions(+), 584 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 0e876f224..cf4590d22 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -70,8 +69,9 @@ namespace Cilium { // Supported verdict kinds using RuleVerdict = enum { None = 0, - Allow = 1, - Deny = 2, + Pass = 1, + Allow = 2, + Deny = 3, }; } // namespace Cilium @@ -95,6 +95,9 @@ template <> struct formatter { case Envoy::Cilium::RuleVerdict::Deny: name = "DENY"; break; + case Envoy::Cilium::RuleVerdict::Pass: + name = "PASS"; + break; default: name = "UNKNOWN"; break; @@ -608,22 +611,25 @@ SniPattern::SniPattern(const Regex::Engine& engine, absl::string_view sni) { class PortNetworkPolicyRule : public Logger::Loggable { public: PortNetworkPolicyRule() - : name_("default allow rule"), deny_(false), proxy_id_(0), precedence_(0), - tier_last_precedence_(0), mutable_remotes_(false), l7_proto_("") {} + : name_("default allow rule"), verdict_(RuleVerdict::Allow), proxy_id_(0), precedence_(0), + tier_last_precedence_(0), pass_index_(0), l7_proto_("") {} PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, - const cilium::PortNetworkPolicyRule& rule, bool shared_resource) - : name_(rule.name()), deny_(rule.deny()), proxy_id_(uint16_t(rule.proxy_id())), - precedence_(rule.precedence()), tier_last_precedence_(rule.pass_precedence()), - mutable_remotes_(!shared_resource), l7_proto_(rule.l7_proto()) { + const cilium::PortNetworkPolicyRule& rule) + : name_(rule.name()), + verdict_(rule.pass_precedence() ? RuleVerdict::Pass + : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), + proxy_id_(uint16_t(rule.proxy_id())), precedence_(rule.precedence()), + tier_last_precedence_(rule.pass_precedence()), pass_index_(0), + l7_proto_(rule.l7_proto()) { if (tier_last_precedence_ > precedence_) { throw EnvoyException( fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } for (const auto& remote : rule.remote_policies()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", - deny_ ? "Denying" : "Allowing", remote, name_); + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, + remote, name_); remotes_.emplace(remote); } if (rule.has_downstream_tls_context()) { @@ -637,7 +643,8 @@ class PortNetworkPolicyRule : public Logger::Loggable { std::make_unique(parent.transportFactoryContext(), config); } for (const auto& sni : rule.server_names()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): Allowing SNI {} by rule {}", sni, name_); + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} SNI {} by rule {}", verdict_, sni, + name_); allowed_snis_.emplace_back(parent.regexEngine(), sni); } if (rule.has_http_rules()) { @@ -671,15 +678,15 @@ class PortNetworkPolicyRule : public Logger::Loggable { RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { // proxy_id must match if we have any. - if (proxy_id_ != 0 && proxy_id != proxy_id_) { + if (proxy_id_ && proxy_id != proxy_id_) { return RuleVerdict::None; } // Remote ID must match if we have any. if (!isRemoteWildcard() && !remotes_.contains(remote_id)) { return RuleVerdict::None; // no verdict } - // Allow rules allow by default when remotes_ is empty, deny rules do not - return deny_ ? RuleVerdict::Deny : RuleVerdict::Allow; + ASSERT(verdict_ != RuleVerdict::None, "rule must have a verdict"); + return verdict_; } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { @@ -795,7 +802,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { if (!name_.empty()) { res.append(indent, ' ').append("name: \"").append(name_).append("\"\n"); } - if (deny_) { + if (verdict_ == RuleVerdict::Deny) { res.append(indent, ' ').append("deny: true\n"); } if (precedence_) { @@ -805,7 +812,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { res.append(indent, ' ') .append(fmt::format("tier_last_precedence: {}\n", tier_last_precedence_)); } - if (proxy_id_ != 0) { + if (proxy_id_) { res.append(indent, ' ').append(fmt::format("proxy_id: {}\n", proxy_id_)); } @@ -851,12 +858,12 @@ class PortNetworkPolicyRule : public Logger::Loggable { DownstreamTLSContextSharedPtr server_context_; UpstreamTLSContextSharedPtr client_context_; bool has_headermatches_{false}; - bool deny_; - uint16_t proxy_id_; + const RuleVerdict verdict_; + const uint16_t proxy_id_; uint32_t precedence_; uint32_t tier_last_precedence_; + uint32_t pass_index_; absl::btree_set remotes_; - bool mutable_remotes_; std::vector allowed_snis_; // All SNIs allowed if empty. std::shared_ptr> @@ -899,7 +906,7 @@ class PortNetworkPolicyRules : public Logger::Loggable { if (rule->has_headermatches_) { can_short_circuit_ = false; } - if (rule->tier_last_precedence_ != 0) { + if (rule->tier_last_precedence_) { has_pass_rules_ = true; } } @@ -908,16 +915,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // append merges 'rules' to 'rules_' by placing the new 'rules' to the end of 'rules_'. // First call marks 'rules_' as initialized. Of further calls, if either is empty, - // we must add a default allow rule to retain the semantics of an empty rules. + // we must add a default allow rule to retain the semantics of empty rules. void append(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules, - bool shared_resource) { + const Protobuf::RepeatedPtrField& rules) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (const auto& it : rules) { - rules_.emplace_back(std::make_shared(parent, it, shared_resource)); + rules_.emplace_back(std::make_shared(parent, it)); updateFor(rules_.back()); } initialized_ = true; @@ -927,48 +933,82 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules, - bool shared_resource) { + const Protobuf::RepeatedPtrField& rules) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { rules_.emplace(rules_.begin(), - std::make_shared(parent, it, shared_resource)); + std::make_shared(parent, it)); updateFor(rules_.front()); } initialized_ = true; } - // appendNonPassRules merges non-pass rules from 'rules' to 'rules_' by placing the new rules to - // the end of 'rules_'. First call marks 'rules_' as initialized. Of further calls, if either is - // empty, we must add a default allow rule to retain the semantics of an empty rules. - void appendNonPassRules(const std::vector& rules) { + // appendRules merges all rules from 'rules' to the end of 'rules_'. + // First call marks 'rules_' as initialized. Of further calls, if either is empty, + // we must add a default allow rule to retain the semantics of the combined rules. + void appendRules(const std::vector& rules) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (auto& rule : rules) { - if (rule->tier_last_precedence_ == 0) { - rules_.insert(rules_.end(), rule); - updateFor(rule); - } + rules_.insert(rules_.end(), rule); + updateFor(rule); } initialized_ = true; } - // sort by descending precedence, retaining the original order within each precedence level + // Sort by descending precedence. Within the same precedence, deny rules come first, + // then allow rules, and pass rules last. This lets runtime pass handling jump + // immediately, as any same-precedence allow/deny verdict has already been seen. void sort() { - // sortRules(rules_); std::stable_sort(rules_.begin(), rules_.end(), [](const PortNetworkPolicyRuleConstSharedPtr& a, const PortNetworkPolicyRuleConstSharedPtr& b) { return (a->precedence_ > b->precedence_) || - (a->precedence_ == b->precedence_ && (a->deny_ && !b->deny_)); + (a->precedence_ == b->precedence_ && a->verdict_ > b->verdict_); }); } + void prepareRuntimePasses() { + if (!has_pass_rules_) { + return; + } + + uint32_t pass_precedence = 0; + for (uint32_t idx = 0; idx < rules_.size(); idx++) { + if (rules_[idx]->tier_last_precedence_ == 0) { + continue; + } + + if (rules_[idx].use_count() > 1) { + // Pass continuation index is specific to this ordered rule set, so shared pass + // rules must be cloned before storing the computed continuation on the rule. + rules_[idx] = std::make_shared(*rules_[idx]); + } + auto& rule = const_cast(*rules_[idx]); + + if (pass_precedence && rule.precedence_ < pass_precedence) { + pass_precedence = 0; + } + + if (pass_precedence && rule.tier_last_precedence_ != pass_precedence) { + throw EnvoyException(fmt::format("PortNetworkPolicy: Inconsistent pass precedence {} != {}", + rule.tier_last_precedence_, pass_precedence)); + } + pass_precedence = rule.tier_last_precedence_; + + uint32_t pass_index = idx + 1; + while (pass_index < rules_.size() && rules_[pass_index]->precedence_ >= pass_precedence) { + pass_index++; + } + rule.pass_index_ = pass_index; + } + } + bool empty() const { return rules_.empty(); } template RuleVerdict forEachRule(bool can_short_circuit, F&& func) const { @@ -986,21 +1026,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return RuleVerdict::Allow; } - for (const auto& rule : rules_) { + for (uint32_t idx = 0; idx < rules_.size();) { + const auto& rule = rules_[idx]; + // lower precedence rules are skipped if there is a verdict if (verdict != RuleVerdict::None && rule->precedence_ < verdict_precedence) { break; } auto rule_verdict = func(*rule); - if (rule_verdict != RuleVerdict::None) { + if (rule_verdict == RuleVerdict::Pass) { + ASSERT(rule->pass_index_, "matching pass rule must have a continuation index"); + if (verdict == RuleVerdict::None || verdict_precedence < rule->precedence_) { + idx = rule->pass_index_; + continue; + } + } else if (rule_verdict != RuleVerdict::None) { verdict = rule_verdict; verdict_precedence = rule->precedence_; // Short-circuit on the first deny or on first allow if no rules have HeaderMatches if (rule_verdict == RuleVerdict::Deny || can_short_circuit) { - break; + return verdict; } } + idx++; } return verdict; } @@ -1026,9 +1075,18 @@ class PortNetworkPolicyRules : public Logger::Loggable { return RuleVerdict::Allow; } - for (const auto& rule : rules_) { + for (uint32_t idx = 0; idx < rules_.size();) { + const auto& rule = rules_[idx]; + auto rule_verdict = get_verdict(*rule); switch (rule_verdict) { + case RuleVerdict::Pass: + ASSERT(rule->pass_index_, "matching pass rule must have a continuation index"); + if (verdict == RuleVerdict::None || verdict_precedence < rule->precedence_) { + idx = rule->pass_index_; + continue; + } + break; case RuleVerdict::Deny: // return higher precedence allow verdict if any. if (verdict != RuleVerdict::None && verdict_precedence > rule->precedence_) { @@ -1049,6 +1107,7 @@ class PortNetworkPolicyRules : public Logger::Loggable { case RuleVerdict::None: break; } + idx++; } return verdict; } @@ -1112,7 +1171,7 @@ class PortNetworkPolicyRules : public Logger::Loggable { void toString(int indent, std::string& res) const { res.append(indent - 2, ' ').append("- rules:\n"); - for (auto& rule : rules_) { + for (const auto& rule : rules_) { rule->toString(indent + 2, res); } if (!can_short_circuit_) { @@ -1129,6 +1188,11 @@ class PortNetworkPolicyRules : public Logger::Loggable { return false; } + bool hasOnlyPassRules() const { + return !rules_.empty() && + std::ranges::all_of(rules_, [](const auto& rule) { return rule->pass_index_ != 0; }); + } + // ordered set of rules as a sorted vector std::vector rules_; // Allowed if empty. bool can_short_circuit_{true}; @@ -1262,449 +1326,8 @@ bool inline rangesOverlap(const PortRange& a, const PortRange& b) { // !(a.second < b.first || a.first > b.second) return a.second >= b.first && a.first <= b.second; } - -template -absl::btree_set intersection(const absl::btree_set& a, const absl::btree_set& b) { - absl::btree_set result; - std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), - std::inserter(result, result.begin())); - return result; -} } // namespace -// ShadowedRemotes maintains state for shadowed remote identities within a tier. When a higher -// precedence rule has a verdict for a given remote identity, that identity becomes "shadowed" and -// is removed from the set of remote identities of the remaining rules of the tier. For pass -// verdicts this shadowing is immediate, for allow/deny verdicts the shadowing takes place for the -// next precedence level, so that rules on the same precedence level do not shadow each other. -class ShadowedRemotes { -public: - // reset is used to re-initialize state for a new port range - void reset(uint32_t first_precedence) { - shadowed_pass_remotes_.clear(); - shadowed_nonpass_remotes_.clear(); - current_precedence_nonpass_remotes_.clear(); - shadow_all_lower_precedence_ = false; - current_precedence_has_wildcard_ = false; - previous_precedence_ = first_precedence; - } - - // resetForNewTier is used to re-initialize state for each new tier - void resetForNewTier(uint32_t first_precedence) { - shadowed_pass_remotes_.clear(); - // shadowed_nonpass_remotes_ are kept for the new tier - // shadow_all_lower_precedence_ is not reset for the new tier - current_precedence_nonpass_remotes_.clear(); - // current_precedence_has_wildcard_ = false; - previous_precedence_ = first_precedence; - } - - // shadowRemotes marks 'remotes' as shadowed and returns 'true' if any of them were not already - // shadowed. - bool shadowPassRemotes(const absl::btree_set& pass_remotes) { - bool any_unshadowed = false; - for (auto remote : pass_remotes) { - if (!shadowed_pass_remotes_.contains(remote)) { - if (!shadowed_nonpass_remotes_.contains(remote)) { - any_unshadowed = true; - } - shadowed_pass_remotes_.insert(remote); - } - } - return any_unshadowed; - } - - // filterShadowPassRemotes filters out any already shadowed remotes from the pass rule and marks - // the remaining remotes as shadowed. Returns 'true' if any remotes remain. - bool filterShadowPassRemotes(PortNetworkPolicyRule& rule) { - absl::erase_if(rule.remotes_, [&](const auto& x) { - return shadowed_pass_remotes_.contains(x) || shadowed_nonpass_remotes_.contains(x); - }); - if (rule.remotes_.empty()) { - return false; - } - shadowed_pass_remotes_.insert(rule.remotes_.begin(), rule.remotes_.end()); - return true; - } - - // shadowRule collects the remotes from 'rule' for shadowing once the first rule on the next - // (lower) precedence level is processed. No shadowing between rules on the same precedence - // level. Returns 'true' if the rule itself should be skipped. - bool shadowNonpassRule(PortNetworkPolicyRule& rule) { - // Same-precedence allow/deny rules do not shadow each other. - // Only after leaving a precedence level do its verdict identities shadow - // lower-precedence rules. - if (rule.precedence_ != previous_precedence_) { - shadowed_nonpass_remotes_.insert(current_precedence_nonpass_remotes_.begin(), - current_precedence_nonpass_remotes_.end()); - current_precedence_nonpass_remotes_.clear(); - if (current_precedence_has_wildcard_) { - shadow_all_lower_precedence_ = true; - } - // current_precedence_has_wildcard_ = false; - previous_precedence_ = rule.precedence_; - } - - if (shadow_all_lower_precedence_) { - return true; - } - - if (rule.isRemoteWildcard()) { - current_precedence_has_wildcard_ = true; - } else { - if (rule.mutable_remotes_) { - // Check if this rule is (partially) shadowed by nonpass rules on any higher tier. - if (!shadowed_nonpass_remotes_.empty()) { - absl::erase_if(rule.remotes_, - [&](const auto& x) { return shadowed_nonpass_remotes_.contains(x); }); - if (rule.remotes_.empty()) { - return true; - } - } - - // Check if this rule is (partially) shadowed by pass rules on this tier. - if (!shadowed_pass_remotes_.empty()) { - absl::erase_if(rule.remotes_, - [&](const auto& x) { return shadowed_pass_remotes_.contains(x); }); - if (rule.remotes_.empty()) { - return true; - } - } - } else { - // rule.remotes_ can not be modified, check if it is completely shadowed - if (std::ranges::all_of(rule.remotes_, [&](const auto& x) { - return shadowed_nonpass_remotes_.contains(x) || shadowed_pass_remotes_.contains(x); - })) { - return true; - } - } - - // Defer shadowing identities until this precedence level is complete, - // so same-precedence rules do not shadow each other. - current_precedence_nonpass_remotes_.insert(rule.remotes_.begin(), rule.remotes_.end()); - } - - return false; - } - -private: - absl::btree_set shadowed_pass_remotes_; - absl::btree_set shadowed_nonpass_remotes_; - absl::btree_set current_precedence_nonpass_remotes_; - uint32_t previous_precedence_{0}; - bool shadow_all_lower_precedence_{false}; - bool current_precedence_has_wildcard_{false}; -}; - -class Passes { -public: - // reset state for a new port range - void reset(uint32_t first_precedence) { - wildcard_it_ = wildcard_pass_rules_.begin(); - pass_rules_.clear(); - pass_rules_tier_index_.clear(); - - tier_pass_rules_.clear(); - tier_wildcard_pass_ = false; - pass_precedence_ = 0; - - shadowing_.reset(first_precedence); - } - - void resetForNextTier(uint32_t first_precedence) { - // Move the collected pass rules to be considered for lower tiers. - pass_rules_tier_index_.push_back(pass_rules_.size()); - pass_rules_.insert(pass_rules_.end(), std::make_move_iterator(tier_pass_rules_.begin()), - std::make_move_iterator(tier_pass_rules_.end())); - - tier_pass_rules_.clear(); - tier_wildcard_pass_ = false; - pass_precedence_ = 0; - - shadowing_.resetForNewTier(first_precedence); - } - - void inheritHigherTierWildcardPassRules(uint32_t precedence) { - // Inherit *higher tier* pass rules from the wildcard port. - // Needed since the wildcard port may have more tiers. - for (; wildcard_it_ != wildcard_pass_rules_.end() && - (*wildcard_it_)->tier_last_precedence_ > precedence; - wildcard_it_++) { - const auto& wildcard_rule = *wildcard_it_; - // Add index entry if this starts a new pass tier. - if (pass_rules_.empty() || - wildcard_rule->tier_last_precedence_ != pass_rules_.back()->tier_last_precedence_) { - pass_rules_tier_index_.push_back(pass_rules_.size()); - } - pass_rules_.insert(pass_rules_.end(), wildcard_rule); - } - } - - void inheritCurrentTierWildcardPassRules(uint32_t precedence) { - // Inherit *current tier* higher or equal precedence pass rules from wildcard port. - for (; - wildcard_it_ != wildcard_pass_rules_.end() && (*wildcard_it_)->precedence_ >= precedence && - (*wildcard_it_)->tier_last_precedence_ <= precedence; - wildcard_it_++) { - const auto& wildcard_rule = *wildcard_it_; - - ensurePassPrecedence(wildcard_rule->tier_last_precedence_); - - if (tier_wildcard_pass_) { - continue; - } - - // Insert to tier_pass_rules_ if any unshadowed remotes remain. - if (wildcard_rule->isRemoteWildcard()) { - tier_wildcard_pass_ = true; - } else if (!shadowing_.shadowPassRemotes(wildcard_rule->remotes_)) { - // Only insert if some remotes are not already shadowed. - // We do not remove already-shadowed remotes from wildcard_rule because - // wildcard pass entries are shared and deep-copying here is expensive. - continue; - } - tier_pass_rules_.emplace_back(wildcard_rule); - } - } - - // addPassRule adds state from a rule with a pass verdict. - void addPassRule(const PortNetworkPolicyRuleConstSharedPtr& rule) { - ensurePassPrecedence(rule->tier_last_precedence_); - - auto& mutable_rule = const_cast(*rule); - - if (!tier_wildcard_pass_) { - if (mutable_rule.isRemoteWildcard()) { - tier_wildcard_pass_ = true; - } else if (!shadowing_.filterShadowPassRemotes(mutable_rule)) { - return; - } - if (!tier_pass_rules_.empty() && tier_pass_rules_.back()->precedence_ == rule->precedence_) { - // Same-precedence pass rule already exists; merge remotes. - tier_pass_rules_.back()->remotes_.insert(mutable_rule.remotes_.begin(), - mutable_rule.remotes_.end()); - } else { - tier_pass_rules_.emplace_back(std::const_pointer_cast(rule)); - } - } - } - - bool - promoteRuleFromHigherTierPasses(std::vector& rules, - std::vector::iterator& it) { - // Mutable reference to the rule for in-place updates below. - auto& rule = const_cast(**it); - - bool promoted = false; - - // Check if this rule needs to be (partially) promoted due to higher-tier passes: - // - pick highest-precedence pass from each higher tier for each remote ID - // - apply in reverse order of tiers - // - if all remotes are covered, promotion can happen fully in-place. - int tier_end = pass_rules_.size(); - for (int tier_start : pass_rules_tier_index_ | std::views::reverse) { - // Skip pass rules on same or lower tiers. - if (pass_rules_[tier_start]->tier_last_precedence_ < rule.precedence_) { - continue; - } - - for (int idx = tier_start; idx < tier_end; idx++) { - auto& pass_rule = pass_rules_[idx]; - // Whole rule is promoted in-place if pass is wildcard or sets are equal. - if (pass_rule->isRemoteWildcard() || rule.remotes_ == pass_rule->remotes_) { - rule.inheritPassPrecedence(*pass_rule); - promoted = true; - break; // Later pass verdicts on this tier have no effect. - } - - // Pass rule is not wildcard and sets differ. - // If mutable_rule is wildcard, keep original and add promoted clone. - if (rule.isRemoteWildcard()) { - auto new_rule = std::make_shared(rule); - new_rule->remotes_ = pass_rule->remotes_; - new_rule->inheritPassPrecedence(*pass_rule); - it = rules.insert(it, new_rule); - it++; - promoted = true; - continue; // Later pass verdicts may specify other remote sets. - } - - // Neither side is wildcard; split by set intersection. - auto remotes = intersection(pass_rule->remotes_, rule.remotes_); - if (!remotes.empty()) { - auto new_rule = std::make_shared(rule); - new_rule->remotes_ = remotes; - new_rule->inheritPassPrecedence(*pass_rule); - it = rules.insert(it, new_rule); - it++; - promoted = true; - for (const auto& remote : remotes) { - rule.remotes_.erase(remote); - } - } - } - // Update for previous tier, if any. - tier_end = tier_start; - } - - return promoted; - } - - // storeWildcardPassRules stores the current pass rules as wildcard port pass rules to be - // considered when processing non-wildcard port rules. - void storeWildcardPassRules(const PortRange& port_range) { - if (!wildcard_pass_rules_.empty()) { - throw EnvoyException(fmt::format("PortNetworkPolicy: Wildcard port range {}-{}, but " - "wildcard pass rules has already been set", - port_range.first, port_range.second)); - } - wildcard_pass_rules_ = pass_rules_; - - // store also the pass rules for the last tier, as they are not yet included in pass_rules_. - if (pass_precedence_ != 0 && !tier_pass_rules_.empty()) { - wildcard_pass_rules_.insert(wildcard_pass_rules_.end(), - std::make_move_iterator(tier_pass_rules_.begin()), - std::make_move_iterator(tier_pass_rules_.end())); - } - } - - // Applies pass verdicts for rules on a given port range. - // Returns true if resulting rules should be kept, false if the rules became empty. - // - // - a pass verdict rule applies on a given port (range) (can be the wildcard port), - // and a set of remote IDs (L3). If the remote ID set is empty, then it applies to all peers. - // (there is no "wildcard identity" (e.g., '0') in the set of the remote IDs) - // - a pass verdict rule (like a deny verdict rule) has no L7 rule components - // - each pass verdict rule has a specific precedence and pass_precedence, and the function is - // to bypass the remaining lower precedence rules upto the pass_precedence, and to promote the - // priority of the intersecting remote ID set lower precedence rules to immediately follow - // the precedence of the pass verdict rule. - // - Precedence promotion is needed to make a verdicts found via actual port vs. wildcard port - // comparable. This allows a higher precedence allow rule to take precedence over a lower - // precedence deny rule, even it the allow originally had a lower precedence, but was - // "passed-to" from a higher precedece pass rule. - // - // We pre-process the rules accordingly here so that the policy lookup at enforcement time - // does not need to consider pass verdicts at all. The key insights to consider are: - // - rules are already split up to non-overlapping port ranges, so we only need - // to consider the remote ID sets (and the wildcard port) - // - if the lower precedence rule remote ID set is covered by the pass rule remote ID set, - // then we can simply promote the precedence (and re-sort afterwards) - // - if the pass rule applies to all remote IDs (empty set == wildcard), then it covers all - // possible sets of remote IDs - // - if not wildcard, but the sets are the same, then the pass verdicts "covers" the rule - // in question - // - otherwise the rule needs to be split into two: - // - one with the intersection of the remote IDs of the two rules, with precedence promotion - // - other with the remaining remote IDs, left with the original precedence - // - this includes the case where the lower precedence rule applies to all identities - // (empty ID set) - void apply(const PortRange& port_range, PortNetworkPolicyRules& rules) { - if (rules.rules_.empty() && !wildcard_pass_rules_.empty()) { - // add the default allow rule so that the wildcard port pass can apply to it. - rules.addDefaultAllowRule(); - } - if (!rules.rules_.empty() && (!wildcard_pass_rules_.empty() || rules.has_pass_rules_)) { - bool must_sort = false; - - // reset state for the new range's rules - reset(rules.rules_.front()->precedence_); - - bool keep = false; // assume rule is dropped - for (auto it = rules.rules_.begin(); it != rules.rules_.end(); - it = keep ? (keep = false, ++it) : rules.rules_.erase(it)) { - auto& rule = *it; - - // Check if we have reached the next tier. - if (pass_precedence_ != 0 && rule->precedence_ < pass_precedence_) { - resetForNextTier(rule->precedence_); - } - - // Skip remaining rules on this tier? - if (tier_wildcard_pass_) { - continue; - } - - // Inherit wildcard-port pass rules affecting this rule. - inheritHigherTierWildcardPassRules(rule->precedence_); - inheritCurrentTierWildcardPassRules(rule->precedence_); - - // skip remaining rules on this tier? - if (tier_wildcard_pass_) { - continue; - } - - // Is this a pass verdict rule? - if (rule->tier_last_precedence_ != 0) { - addPassRule(rule); - // Pass rules are not kept - continue; - } - - // Is the rule shadowed? (If not then updates shadowed state) - if (shadowing_.shadowNonpassRule(const_cast(*rule))) { - continue; - } - - // Apply passes to the rule and insert. - if (promoteRuleFromHigherTierPasses(rules.rules_, it)) { - must_sort = true; - } - // keep rule in place - keep = true; - } - - // Have to sort if precedences have been updated in-place. - if (must_sort) { - rules.sort(); - - // remove shadowed rules due to promoted precedences - shadowing_.reset(rules.rules_.front()->precedence_); - for (auto it = rules.rules_.begin(); it != rules.rules_.end(); - it = keep ? ++it : rules.rules_.erase(it)) { - keep = !shadowing_.shadowNonpassRule(const_cast(**it)); - } - } - - // Store wildcard port passes for consideration for non-wildcard ports. - if (port_range.first == 0) { - storeWildcardPassRules(port_range); - } - - // Mark ranges with all rules removed for clean-up. - if (rules.empty()) { - // Empty rule set would always allow. Mark for removal. - empty_ranges_.push_back(port_range); - } - } - } - - std::vector& emptyRanges() { return empty_ranges_; } - -private: - void ensurePassPrecedence(uint32_t tier_last_precedence) { - // pass_precedence_ is non-zero when a pass verdict has been seen and - // defines the end of the current tier. All pass verdicts on a specific - // tier must have the same pass_precedence so tier boundaries stay unambiguous. - if (tier_last_precedence == 0 || - (pass_precedence_ != 0 && tier_last_precedence != pass_precedence_)) { - throw EnvoyException(fmt::format("PortNetworkPolicy: Inconsistent pass precedence {} != {}", - tier_last_precedence, pass_precedence_)); - } - pass_precedence_ = tier_last_precedence; - } - - std::vector empty_ranges_; - std::vector wildcard_pass_rules_; - std::vector::iterator wildcard_it_; - ShadowedRemotes shadowing_; - std::vector pass_rules_; - std::vector pass_rules_tier_index_; - uint32_t pass_precedence_{0}; - std::vector tier_pass_rules_; - bool tier_wildcard_pass_{false}; -}; - class PortNetworkPolicy : public Logger::Loggable { public: PortNetworkPolicy(const NetworkPolicyMapImpl& parent, @@ -1717,7 +1340,7 @@ class PortNetworkPolicy : public Logger::Loggable { // End port may be zero, which means no range uint16_t end_port = rule.end_port(); if (end_port < port) { - if (end_port != 0) { + if (end_port) { throw EnvoyException(fmt::format( "PortNetworkPolicy: Invalid port range, end port is less than start port {}-{}", port, end_port)); @@ -1853,7 +1476,6 @@ class PortNetworkPolicy : public Logger::Loggable { RELEASE_ASSERT(it != rules_.end(), "first overlapping entry not found"); } // Add rules to all the overlapping entries - bool shared_resource = rule_range.first == 0; // wildcard port rules are shared bool singular = rule_range.first == rule_range.second; for (; it != rules_.end() && rangesOverlap(it->first, rule_range); it++) { auto range = it->first; @@ -1868,10 +1490,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules(), shared_resource); + rules.prepend(parent, rule.rules()); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules(), shared_resource); + rules.append(parent, rule.rules()); } } } else { @@ -1889,7 +1511,7 @@ class PortNetworkPolicy : public Logger::Loggable { if (!wildcard_rules) { break; } - rules.appendNonPassRules(wildcard_rules->rules_); + rules.appendRules(wildcard_rules->rules_); } bool have_passes = false; @@ -1905,19 +1527,20 @@ class PortNetworkPolicy : public Logger::Loggable { } } - // Apply pass verdicts, if any. if (have_passes) { - Passes passes; - - // This loop always iterates the wildcard port first, if rules for it exist. for (auto& [port_range, rules] : rules_) { - passes.apply(port_range, rules); + (void)port_range; + rules.prepareRuntimePasses(); } - // Delete port ranges that only contained pass rules. - // Otherwise the policy would always accept. - for (auto port_range : passes.emptyRanges()) { - rules_.erase(port_range); + // Pass-only ranges do not yield a final verdict, but keeping them would make + // higher-level L7 checks treat the range as "no L7 rules" and allow by default. + for (auto it = rules_.begin(); it != rules_.end();) { + if (it->second.hasOnlyPassRules()) { + it = rules_.erase(it); + } else { + ++it; + } } } } diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index f05ee3993..222fad4e9 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -1189,9 +1189,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 10 + tier_last_precedence: 1 - remotes: [] name: "default allow rule" - precedence: 9 egress: rules: [] )EOF"; @@ -1231,9 +1233,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 10 + tier_last_precedence: 1 - remotes: [] name: "default allow rule" - precedence: 9 egress: rules: [] )EOF"; @@ -1402,14 +1406,17 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1461,14 +1468,17 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43,44] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1520,8 +1530,14 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 - remotes: [43,44] - precedence: 999 + precedence: 500 http_rules: - headers: - name: ":path" @@ -1578,21 +1594,24 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected9)); - // Remote 43 is promoted above deny by pass. + // Remote 43 matches the pass rule and skips the intermediate deny. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); // Other remotes are still denied by the deny rule. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); @@ -1633,23 +1652,26 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 + - remotes: [43,44] + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected10)); - // Pass from wildcard port should promote remote 43 above deny on port 80. + // Pass from wildcard port lets remote 43 skip the deny on port 80. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); - // Remote 44 is denied due to only 43 being promoted. + // Remote 44 is denied because only 43 matches the pass rule. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); // Unspecified remotes remain denied. EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allowed"}})); @@ -1694,15 +1716,21 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [43] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [44] + precedence: 1000 + tier_last_precedence: 501 + - remotes: [] + deny: true + precedence: 900 - remotes: [43,44] - precedence: 999 + precedence: 500 http_rules: - headers: - name: ":path" value: "/allowed" - - remotes: [] - deny: true - precedence: 900 egress: rules: [] )EOF"; @@ -1715,15 +1743,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allowed"}})); // - // 12th update: non-pass rule shadowing inside a pass tier + // 12th update: pass tier keeps native rules in runtime order // - // The pass rule is required to enable tier processing, but it targets only - // remote 45 so the tier is not wildcard-pass and does not pre-shadow 43/44. - // Within this tier: - // - A higher-precedence deny for remote 44 establishes a final verdict for 44. - // - A lower-precedence allow for [43,44] must have 44 removed due to shadowing. - // - A second allow at the same precedence for [43] must keep 43, confirming - // no same-precedence identity shadowing between allow rules. + // With runtime pass handling the pass rule remains present, the higher-precedence + // deny for remote 44 stays in place, and the lower allow rules are evaluated in + // their native form instead of being rewritten during preprocessing. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "12" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1768,11 +1792,8 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [45] - precedence: 999 - http_rules: - - headers: - - name: ":path" - value: "/allow-c" + precedence: 1000 + tier_last_precedence: 701 - remotes: [44] deny: true precedence: 900 @@ -1782,7 +1803,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { - headers: - name: ":path" value: "/allow-b" - - remotes: [43] + - remotes: [43,44] precedence: 800 http_rules: - headers: @@ -1803,12 +1824,12 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { // Remote 43 is not passed, but both same-precedence allow rules remain effective. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allow-a"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allow-b"}})); - // Remote 44 is denied by the higher-precedence deny and removed from allow-a. + // Remote 44 is denied by the higher-precedence deny before either allow rule is reached. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-a"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-b"}})); - // Pass remote 45 does not match /allow-a because only /allow-c is promoted for it. + // Pass remote 45 does not match /allow-a because only the lower wildcard allow matches it. EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allow-a"}})); - // Wildcard allow at precedence 700 is promoted to precedence 999 only for pass remote 45. + // Pass remote 45 reaches the lower wildcard allow at precedence 700. EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/allow-c"}})); // Non-pass remotes not already denied at higher precedence still match the // original wildcard rule at precedence 700. @@ -1816,12 +1837,11 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allow-c"}})); // - // 13th update: inherited wildcard current-tier pass fully shadowed + // 13th update: inherited wildcard and local pass rules both remain // - // Wildcard port has a current-tier pass for remote 43, and specific port has - // a higher precedence pass for the same remote on the same tier. When the - // wildcard pass is inherited, it is fully shadowed and skipped, as evidenced by the - // precedence of the passed-to rule for remote 43, which is 999 rather than 899. + // Runtime pass handling keeps both pass rules visible in precedence order. The + // specific-port pass is checked before the inherited wildcard-port pass for the + // same remote, but neither rule is rewritten away. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "13" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1857,39 +1877,38 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [43] - precedence: 999 + precedence: 1000 + tier_last_precedence: 701 + - remotes: [43] + precedence: 900 + tier_last_precedence: 701 + - remotes: [] + deny: true + precedence: 800 + - remotes: [43,44] + precedence: 700 http_rules: - headers: - name: ":path" value: "/shadowed-inherited-pass" - - remotes: [] - deny: true - precedence: 800 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected13)); - // Remote 43 is promoted above deny due to the specific-port pass. + // Remote 43 hits the specific-port pass before the intermediate deny. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/shadowed-inherited-pass"}})); // Remote 44 remains denied by the intermediate deny. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/shadowed-inherited-pass"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/shadowed-inherited-pass"}})); // - // 14th update: multiple wildcard pass tiers inherited by a specific port + // 14th update: multiple inherited pass tiers are handled at runtime // - // Wildcard port contributes two pass tiers: - // Tier boundaries are inclusive. - // - tier 1 pass (1300/1000) for remote 41: tier boundaries [1300..1000] - // - tier 2 pass (900/700) for remote 42: tier boundaries [999..700] - // For port 80: - // - deny at 850 is within tier 2, so it is promoted by tier 1 pass for remote 41 to 1150 - // - allow [41,42,43] at 600 is split and promoted by both tiers: - // - 41 to tier 1 precedence 900 - // - 42 to tier 2 precedence 800 - // - 43 remains at tier 3 at precedence 600 + // The wildcard-port pass rules remain visible in the exact-port rule list. Remote 41 + // still reaches the intermediate deny, while remote 42 matches the lower wildcard pass + // and skips that deny to the allow at precedence 600. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "14" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -1925,28 +1944,31 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { [80-80]: - rules: - remotes: [41] - deny: true - precedence: 1050 + precedence: 1300 + tier_last_precedence: 1000 - remotes: [42] - precedence: 800 + precedence: 900 + tier_last_precedence: 700 + - remotes: [] + deny: true + precedence: 750 + - remotes: [41,42,43] + precedence: 600 http_rules: - headers: - name: ":path" value: "/multi-tier" - - remotes: [] - deny: true - precedence: 750 egress: rules: [] )EOF"; EXPECT_TRUE(validate("10.1.2.3", expected14)); - // Remote 41 hits the promoted deny from tier 1. + // Remote 41 does not match the lower pass tier, so it still hits the deny at 850. EXPECT_FALSE(ingressAllowed("10.1.2.3", 41, 80, {{":path", "/multi-tier"}})); - // Remote 42 is promoted by the lower wildcard tier + // Remote 42 matches the lower pass tier and skips the deny to the allow at precedence 600. EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); - // Remote 43 is not promoted and is denied. + // Remote 43 does not match either pass tier and is denied. EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/multi-tier"}})); // @@ -1989,11 +2011,10 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_TRUE(ingressAllowed("10.1.2.3", 42, 80, {{":path", "/multi-tier"}})); // - // 16th update: inherited wildcard pass skips remaining rules on that tier + // 16th update: inherited wildcard pass skips remaining rules on that tier at runtime // - // Wildcard port has a wildcard pass (2000/700), which is inherited for port 80. - // Rules in that same tier [1999..700] are skipped; a lower-tier rule at 600 is - // retained and promoted to 1900 by the inherited wildcard pass. + // The pass rule and the skipped-tier rules remain present in the rendered policy, but a + // matching remote jumps past the 1200/1100 rules and reaches the lower-tier allow at 600. EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "16" resources: - "@type": type.googleapis.com/cilium.NetworkPolicy @@ -2032,8 +2053,20 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 2000 + tier_last_precedence: 700 + - remotes: [43] + deny: true + precedence: 1200 + - remotes: [44] + precedence: 1100 + http_rules: + - headers: + - name: ":path" + value: "/should-skip" - remotes: [43,44] - precedence: 1900 + precedence: 600 http_rules: - headers: - name: ":path" @@ -2044,15 +2077,15 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_TRUE(validate("10.1.2.3", expected16)); - // Both remotes are allowed by the promoted lower-tier rule. + // Both remotes are allowed by the lower-tier rule reached after the wildcard pass. EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/promoted-after-skip"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/promoted-after-skip"}})); - // Tier rule at 800 is skipped by inherited wildcard pass. + // The 1100 rule remains present but is skipped by the inherited wildcard pass. EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/should-skip"}})); EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/promoted-after-skip"}})); // - // 17th update: Shadowed rules are eliminated + // 17th update: overlapping lower rules remain visible under runtime pass handling // EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "17" resources: @@ -2092,11 +2125,20 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { rules: [80-80]: - rules: + - remotes: [] + precedence: 1000 + tier_last_precedence: 901 - remotes: [43] deny: true - precedence: 999 - - remotes: [44] - precedence: 699 + precedence: 900 + - remotes: [43] + precedence: 800 + http_rules: + - headers: + - name: ":path" + value: "/should-skip" + - remotes: [43,44] + precedence: 600 http_rules: - headers: - name: ":path" @@ -2109,7 +2151,7 @@ TEST_F(CiliumNetworkPolicyTest, Precedence) { EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/partially-skipped"}})); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/partially-skipped"}})); - // Rule at 800 is shadowed by higher precedence deny + // Rule at 800 remains present, but the higher-precedence deny still wins for remote 43. EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/should-skip"}})); // inapplicable identity EXPECT_FALSE(ingressAllowed("10.1.2.3", 45, 80, {{":path", "/partially-skipped"}})); From 7ef3b32ec060e858fae86fe3c38829c82fdefe18 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sat, 4 Apr 2026 21:51:25 +0200 Subject: [PATCH 07/14] policy: Add NetworkPolicyResourcesDiscoveryService Add new cilium/versioned.h generic container for transactional selector updates. Add a new NetworkPolicyResourceDiscoveryService that implements delta updates for policies and selectors, and where policies refer to selectors by their resource name. NPRDS adds a top-level oneof wrapper that wraps either a Selector or a NetworkPolicy. NetworkPolicy definition is shared with NPDS, but PortNetworkPolicyRule adds a new selectors field that is only used with NPRDS. Signed-off-by: Jarno Rajahalme --- cilium/BUILD | 12 + cilium/api/bpf_metadata.proto | 6 + cilium/api/npds.proto | 33 + cilium/bpf_metadata.cc | 10 +- cilium/grpc_subscription.cc | 21 +- cilium/grpc_subscription.h | 4 +- cilium/network_policy.cc | 1050 +++++++++++++-- cilium/network_policy.h | 19 +- cilium/versioned.h | 518 ++++++++ go/cilium/api/bpf_metadata.pb.go | 45 +- go/cilium/api/bpf_metadata.pb.validate.go | 31 + go/cilium/api/npds.pb.go | 357 ++++-- go/cilium/api/npds.pb.validate.go | 288 +++++ go/cilium/api/npds_grpc.pb.go | 103 ++ tests/BUILD | 9 + tests/cilium_network_policy_test.cc | 1414 ++++++++++++++++++++- tests/versioned_test.cc | 1165 +++++++++++++++++ 17 files changed, 4836 insertions(+), 249 deletions(-) create mode 100644 cilium/versioned.h create mode 100644 tests/versioned_test.cc diff --git a/cilium/BUILD b/cilium/BUILD index b9572b8a5..e52aa33eb 100644 --- a/cilium/BUILD +++ b/cilium/BUILD @@ -28,6 +28,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "versioned_lib", + hdrs = ["versioned.h"], + repository = "@envoy", + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@envoy//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "network_policy_lib", srcs = [ @@ -45,6 +56,7 @@ envoy_cc_library( "//cilium:conntrack_lib", "//cilium:grpc_subscription_lib", "//cilium:ipcache_lib", + "//cilium:versioned_lib", "//cilium/api:npds_cc_proto", "@envoy//envoy/config:subscription_interface", "@envoy//envoy/singleton:manager_interface", diff --git a/cilium/api/bpf_metadata.proto b/cilium/api/bpf_metadata.proto index 9e660991c..f30ee835b 100644 --- a/cilium/api/bpf_metadata.proto +++ b/cilium/api/bpf_metadata.proto @@ -86,4 +86,10 @@ message BpfMetadata { // Configuration for the source of NPDS updates. Currently this field is not supported. envoy.config.core.v3.ConfigSource npds_config = 16; + + // Use delta NPDS rather than the state-of-the-world protocol. + // Even with delta NPDS, each new stream starts with a full dump. + // Only wildcard subscriptions are supported. + // All listeners on the node must agree on this setting. + bool use_delta_npds = 17; } diff --git a/cilium/api/npds.proto b/cilium/api/npds.proto index 1a43692e9..b5a1e37ac 100644 --- a/cilium/api/npds.proto +++ b/cilium/api/npds.proto @@ -17,6 +17,7 @@ import "validate/validate.proto"; // [#protodoc-title: Network policy management and NPDS] // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. service NetworkPolicyDiscoveryService { option (envoy.annotations.resource).type = "cilium.NetworkPolicy"; @@ -33,6 +34,32 @@ service NetworkPolicyDiscoveryService { } } +// Policy and selector resource names are exact-match identifiers in delta NPDS. +service NetworkPolicyResourceDiscoveryService { + option (envoy.annotations.resource).type = "cilium.NetworkPolicyResource"; + + rpc DeltaNetworkPolicyResources(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } +} + +// A delta NPDS resource that carries either an endpoint policy or a shared selector. +message NetworkPolicyResource { + oneof resource { + NetworkPolicy policy = 1; + Selector selector = 2; + } +} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +message Selector { + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + repeated uint32 remote_identities = 1; +} + // A network policy that is enforced by a filter on the network flows to/from // associated hosts. message NetworkPolicy { @@ -153,6 +180,12 @@ message PortNetworkPolicyRule { // Optional. If not specified, any remote host is matched by this predicate. repeated uint32 remote_policies = 7; + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + repeated string selectors = 11; + // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. TLSContext downstream_tls_context = 3; diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index 2e787c539..084be3f4f 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -260,8 +260,14 @@ Config::Config(const ::cilium::BpfMetadata& config, if (ipcache_ || hosts_) { npmap_ = context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), - [&context] { return std::make_shared(context, true); }); + SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), [&context, &config] { + return std::make_shared(context, true, + config.use_delta_npds()); + }); + if (npmap_->useDeltaXds() != config.use_delta_npds()) { + throw EnvoyException( + "cilium.bpf_metadata: use_npds_delta must be consistent across listeners"); + } } } diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index bcd11db0c..d9109abad 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -105,6 +105,7 @@ TypeUrlToServiceMap* buildTypeUrlToServiceMap() { // https://www.mail-archive.com/protobuf@googlegroups.com/msg04540.html. for (absl::string_view name : { "cilium.NetworkPolicyDiscoveryService", + "cilium.NetworkPolicyResourceDiscoveryService", "cilium.NetworkPolicyHostsDiscoveryService", }) { const auto* service_desc = @@ -168,9 +169,9 @@ const Protobuf::MethodDescriptor& sotwGrpcMethod(absl::string_view type_url) { // Hard-coded Cilium gRPC cluster // Note: No rate-limit settings are used, consider if needed. -envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig() { +envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig(bool use_delta_xds = false) { auto config_source = envoy::config::core::v3::ConfigSource(); - /* config_source.initial_fetch_timeout is set to 50 millliseconds. + /* config_source.initial_fetch_timeout is set to 50 milliseconds. * This applies only to SDS Secrets for now, as for NPDS and NPHDS we explicitly set the timeout * as 0 (no timeout). */ @@ -178,7 +179,9 @@ envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig() { config_source.set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); auto api_config_source = config_source.mutable_api_config_source(); api_config_source->set_set_node_on_first_message_only(true); - api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); + api_config_source->set_api_type(use_delta_xds + ? envoy::config::core::v3::ApiConfigSource::DELTA_GRPC + : envoy::config::core::v3::ApiConfigSource::GRPC); api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); api_config_source->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name("xds-grpc-cilium"); return config_source; @@ -203,9 +206,9 @@ uint64_t grpcStreamGeneration(Config::Subscription* subscription) { std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, - Config::OpaqueResourceDecoderSharedPtr resource_decoder, + Config::OpaqueResourceDecoderSharedPtr resource_decoder, bool use_delta_xds, std::chrono::milliseconds init_fetch_timeout) { - const envoy::config::core::v3::ConfigSource config_source = getCiliumXDSAPIConfig(); + const envoy::config::core::v3::ConfigSource config_source = getCiliumXDSAPIConfig(use_delta_xds); const envoy::config::core::v3::ApiConfigSource& api_config_source = config_source.api_config_source(); THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( @@ -230,7 +233,7 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte factory_or_error.value()->createUncachedRawAsyncClient(), Grpc::RawAsyncClientPtr), /*failover_async_client_=*/nullptr, /*dispatcher_=*/context.mainThreadDispatcher(), - /*service_method_=*/sotwGrpcMethod(type_url), + /*service_method_=*/use_delta_xds ? deltaGrpcMethod(type_url) : sotwGrpcMethod(type_url), /*local_info_=*/context.localInfo(), /*rate_limit_settings_=*/rate_limit_settings_or_error.value(), /*scope_=*/scope, @@ -246,8 +249,10 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte }; std::shared_ptr grpc_mux = - std::static_pointer_cast(std::make_shared( - grpc_mux_context, api_config_source.set_node_on_first_message_only())); + use_delta_xds ? std::static_pointer_cast( + std::make_shared(grpc_mux_context)) + : std::static_pointer_cast(std::make_shared( + grpc_mux_context, api_config_source.set_node_on_first_message_only())); return std::make_unique( grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index bffa2359e..b66bec600 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -7,7 +7,7 @@ #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/subscription.h" -#include "envoy/server/factory_context.h" +#include "envoy/ssl/context_manager.h" #include "envoy/stats/scope.h" namespace Envoy { @@ -19,7 +19,7 @@ extern envoy::config::core::v3::ConfigSource cilium_xds_api_config; std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, - Config::OpaqueResourceDecoderSharedPtr resource_decoder, + Config::OpaqueResourceDecoderSharedPtr resource_decoder, bool use_delta_xds = false, std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); // Returns a monotonic stream generation for Cilium subscriptions. diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index cf4590d22..15b77947b 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -36,7 +37,6 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/common/matchers.h" #include "source/common/common/thread.h" #include "source/common/http/header_utility.h" @@ -44,6 +44,7 @@ #include "source/common/init/target_impl.h" #include "source/common/init/watcher_impl.h" #include "source/common/network/utility.h" +#include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" #include "source/server/transport_socket_config_impl.h" @@ -57,11 +58,13 @@ #include "absl/strings/match.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "cilium/accesslog.h" #include "cilium/api/npds.pb.h" #include "cilium/grpc_subscription.h" #include "cilium/ipcache.h" #include "cilium/secret_watcher.h" +#include "cilium/versioned.h" namespace Envoy { namespace Cilium { @@ -111,20 +114,265 @@ template <> struct formatter { namespace Envoy { namespace Cilium { +// A specific version of a selector used in a policy. Each update yields a new instance. +class SelectorInstance : public VersionedNode, + public absl::flat_hash_set {}; + +// Read-only series of specific selector insteances. +class NamedSelectorReadable : public VersionedReadable { +public: + explicit NamedSelectorReadable(const std::string& name) : name_(name) {} + + const std::string& name() const { return name_; } + +private: + std::string name_; +}; + +// Stable handle on a read-only selector for accessing specific versions of the selector. +using SelectorHandle = std::shared_ptr; + +// Writable series of selector versions for main-thread updates +class NamedSelectorValue : public VersionedValue { +public: + explicit NamedSelectorValue(const std::string& name) + : VersionedValue(name) {} +}; + +// Map of named selectors, keyed with xDS resource name +class SelectorMap : public VersionedMap { +public: + using VersionedMap::VersionedMap; +}; + class PolicyInstanceImpl; +// Stable policy map, usually keyed with endpoint IP (IPv4 and IPv6). using PolicyMapSnapshot = absl::flat_hash_map>; +// variant wrapper for supported resource map keys for delta policy updates +// Delta xDS refers to removed resources by resource name, so we must have a map to +// locate the policy/selector to be removed. +class ResourceKey { +public: + struct PolicyResourceEntry { + std::shared_ptr policy; + }; + + struct PolicyEndpointIpEntry {}; + + struct SelectorResourceEntry { + SelectorHandle handle; + }; + + static ResourceKey policyResource(const std::shared_ptr& policy) { + return ResourceKey(PolicyResourceEntry{policy}); + } + + static ResourceKey policyEndpointIp() { return ResourceKey(PolicyEndpointIpEntry{}); } + + static ResourceKey selectorResource(const SelectorHandle& handle) { + return ResourceKey(SelectorResourceEntry{handle}); + } + + const PolicyResourceEntry* policyResourceEntry() const { + return absl::get_if(&value_); + } + + const SelectorResourceEntry* selectorResourceEntry() const { + return absl::get_if(&value_); + } + + bool isPolicyEndpointIpEntry() const { + return absl::holds_alternative(value_); + } + +private: + explicit ResourceKey(const PolicyResourceEntry& value) : value_(value) {} + explicit ResourceKey(const PolicyEndpointIpEntry& value) : value_(value) {} + explicit ResourceKey(const SelectorResourceEntry& value) : value_(value) {} + + absl::variant value_; +}; + +// Map of Delta xDS resources for name collision and duplicate name detection. +class ResourceMap : public absl::flat_hash_map { +public: + using absl::flat_hash_map::flat_hash_map; + + const ResourceKey* findEntry(const std::string& key) const { + auto it = find(key); + return it != end() ? &it->second : nullptr; + } + + void replaceWith(std::vector>&& entries) { + clear(); + reserve(entries.size()); + for (auto& [key, value] : entries) { + insert_or_assign(std::move(key), std::move(value)); + } + } + + void erasePolicyResource(PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy); +}; + +// ResourceMapOverlay lets delta updates stage tentative resource-map removals and insertions on top +// of the current ResourceMap while validation is still in progress. This preserves transactional +// behavior without copying the full map: failed updates can be discarded cheaply, and successful +// ones are applied to the real map only after the whole update has been accepted. +class ResourceMapOverlay { +public: + ResourceMapOverlay() = default; + explicit ResourceMapOverlay(const ResourceMap& base) : base_(&base) {} + + const ResourceKey* findEntry(const std::string& key) const { + auto upsert_it = upserts_.find(key); + if (upsert_it != upserts_.end()) { + return &upsert_it->second; + } + if (removed_.contains(key)) { + return nullptr; + } + return base_ ? base_->findEntry(key) : nullptr; + } + + SelectorHandle getSelectorHandleOrThrow(const std::string& selector) const { + const auto* entry = findEntry(selector); + if (entry == nullptr) { + throw EnvoyException(fmt::format( + "Delta Network Policy rule references missing selector resource '{}'", selector)); + } + const auto* selector_entry = entry->selectorResourceEntry(); + if (selector_entry == nullptr || selector_entry->handle == nullptr) { + throw EnvoyException( + fmt::format("Delta Network Policy rule references non-selector resource '{}'", selector)); + } + return selector_entry->handle; + } + + bool emplace(std::string key, ResourceKey value) { + if (findEntry(key)) { + return false; + } + removed_.erase(key); + return upserts_.emplace(std::move(key), std::move(value)).second; + } + + void insertOrAssign(std::string key, ResourceKey value) { + removed_.erase(key); + upserts_.insert_or_assign(std::move(key), std::move(value)); + } + + void erase(const std::string& key) { + upserts_.erase(key); + if (base_ && base_->find(key) != base_->end()) { + removed_.insert(key); + } else { + removed_.erase(key); + } + } + + bool eraseSelectorResourceIfPresent(const std::string& key) { + const auto* entry = findEntry(key); + if (entry == nullptr || entry->selectorResourceEntry() == nullptr) { + return false; + } + erase(key); + return true; + } + + bool erasePolicyResourceIfPresent(PolicyMapSnapshot& policy_map, + const std::string& resource_name) { + const auto* entry = findEntry(resource_name); + if (entry == nullptr) { + return false; + } + const auto* policy_entry = entry->policyResourceEntry(); + if (policy_entry == nullptr) { + return false; + } + erasePolicyResource(policy_map, resource_name, policy_entry->policy); + return true; + } + + void erasePolicyResource(PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy); + + void applyTo(ResourceMap& map) && { + if (!upserts_.empty()) { + map.reserve(map.size() + upserts_.size()); + } + for (const auto& key : removed_) { + map.erase(key); + } + for (auto& [key, value] : upserts_) { + map.insert_or_assign(std::move(key), std::move(value)); + } + } + +private: + const ResourceMap* base_{}; + absl::flat_hash_set removed_; + absl::flat_hash_map upserts_; +}; + +// helper for validating resource names. +void validateResourceNameHasNoWhitespace(absl::string_view resource_name, + absl::string_view subject) { + if (std::ranges::any_of(resource_name, [](unsigned char c) { return absl::ascii_isspace(c); })) { + throw EnvoyException( + fmt::format("{} '{}' must not contain whitespace", subject, resource_name)); + } +} + +// PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. +// Same-stream selector-only updates publish a newer selector version into this object so existing +// policies follow immediately. When the NPDS stream restarts, new policies get a fresh state +// object while old policies keep the old one until the old policy map has quiesced and been +// retired. This allows the new stream to reuse selector resource names so that the xDS server +// need not keep selector resource names in stable storage accross restarts. +class PolicyStreamState { +public: + explicit PolicyStreamState(uint64_t stream_generation, SelectorVersion version = versionMin) + : stream_generation_(stream_generation), version_(version) {} + + uint64_t streamGeneration() const { return stream_generation_; } + + SelectorVersion version() const { return version_.load(std::memory_order_acquire); } + + void publishVersion(SelectorVersion version) { + version_.store(version, std::memory_order_release); + } + +private: + const uint64_t stream_generation_; + std::atomic version_; +}; +using PolicyStreamStateSharedPtr = std::shared_ptr; +using PolicyStreamStateConstSharedPtr = std::shared_ptr; + class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, - public Logger::Loggable { + public Logger::Loggable, + public std::enable_shared_from_this { public: - NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context); + friend class PortNetworkPolicyRule; + NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, bool use_delta_xds); ~NetworkPolicyMapImpl() override; void startSubscription() { - subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, - *npds_stats_scope_, *this, std::make_shared()); + if (use_delta_xds_) { + subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicyResource", context_, + *npds_stats_scope_, *this, + std::make_shared( + ProtobufMessage::getNullValidationVisitor(), "name"), + use_delta_xds_); + } else { + subscription_ = + subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, *npds_stats_scope_, *this, + std::make_shared(), use_delta_xds_); + } } // This is used for testing with a file-based subscription @@ -137,13 +385,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; @@ -155,10 +397,15 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } + bool useDeltaXds() const { return use_delta_xds_; } + protected: - uint64_t streamGeneration() const { return grpcStreamGeneration(subscription_.get()); } + uint64_t streamGeneration() const { + return stream_generation_override_for_test_ != 0 ? stream_generation_override_for_test_ + : grpcStreamGeneration(subscription_.get()); + } - void resetStreamForTest() { applied_stream_generation_ = 0; } + void resetStreamForTest() { stream_generation_override_for_test_ = streamGeneration() + 1; } // run the given function after all the threads have scheduled void runAfterAllThreads(std::function cb) const { @@ -169,18 +416,20 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, context_.threadLocal().runOnAllWorkerThreads([]() {}, cb); } - std::string resourceName(const cilium::NetworkPolicy& config) { - return fmt::format("{}", config.endpoint_id()); - } - void reopenIpcache(); std::shared_ptr - createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); + createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMap& old_resource_map, + const ResourceMapOverlay* selector_resource_map); + + SelectorHandle createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, uint64_t update_version); void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, std::string&& version_name, - uint64_t stream_generation); + const PolicyStreamStateSharedPtr& policy_stream_state); private: // Helpers for atomic swap of the policy map pointer. @@ -222,12 +471,28 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedImpl(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTestImpl(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTestImpl(const PolicyInstance& policy) const; void removeInitManager(); + void scheduleSelectorDeferredDeletion(DeferredDeletion&& deferred); + void scheduleSelectorGCAndDeferredDeletion(uint64_t published_version, + const PolicyMapSnapshot* old_policy_map = nullptr); static uint64_t instance_id_; + const bool use_delta_xds_; Server::Configuration::ServerFactoryContext& context_; + std::atomic map_ptr_; + SelectorMap selector_map_; + // Policies hold a shared per-stream state object. A freshly installed stream stores its actual + // gRPC stream generation here, so same-stream selector-only updates advance existing policies + // immediately while old policies remain pinned to the latest selector version reached by their + // own stream. + PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; + ResourceMap resource_map_; Stats::ScopeSharedPtr npds_stats_scope_; Stats::ScopeSharedPtr policy_stats_scope_; @@ -237,10 +502,9 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, transport_factory_context_; std::unique_ptr subscription_; - // Value 0 is reserved for detection of the initial stream before the first - // successful policy install. Tracked gRPC subscriptions may also report 0 - // before any stream has been established. - uint64_t applied_stream_generation_{0}; + // Test-only override used to simulate a restarted NPDS stream when the test subscription does + // not expose a new underlying gRPC stream generation. + uint64_t stream_generation_override_for_test_{0}; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -253,6 +517,10 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, uint64_t NetworkPolicyMapImpl::instance_id_ = 0; +namespace { +constexpr absl::string_view WildcardResourceName = "*"; +} // namespace + IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { auto ip = Network::Utility::parseInternetAddressNoThrow(ip_addr); @@ -615,22 +883,38 @@ class PortNetworkPolicyRule : public Logger::Loggable { tier_last_precedence_(0), pass_index_(0), l7_proto_("") {} PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, - const cilium::PortNetworkPolicyRule& rule) + const cilium::PortNetworkPolicyRule& rule, + const ResourceMapOverlay* selector_resource_map) : name_(rule.name()), verdict_(rule.pass_precedence() ? RuleVerdict::Pass : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), proxy_id_(uint16_t(rule.proxy_id())), precedence_(rule.precedence()), - tier_last_precedence_(rule.pass_precedence()), pass_index_(0), - l7_proto_(rule.l7_proto()) { + tier_last_precedence_(rule.pass_precedence()), pass_index_(0), l7_proto_(rule.l7_proto()) { if (tier_last_precedence_ > precedence_) { throw EnvoyException( fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } - for (const auto& remote : rule.remote_policies()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, - remote, name_); - remotes_.emplace(remote); + if (selector_resource_map) { + if (rule.remote_policies_size()) { + throw EnvoyException( + "Delta Network Policy rule must use selectors instead of remote_policies"); + } + selectors_.reserve(rule.selectors_size()); + for (const auto& selector : rule.selectors()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} selector {} by rule: {}", verdict_, + selector, name_); + selectors_.emplace_back(selector_resource_map->getSelectorHandleOrThrow(selector)); + } + } else { + if (rule.selectors_size()) { + throw EnvoyException("State-of-the-world Network Policy rule must not use selectors"); + } + for (const auto remote : rule.remote_policies()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, + remote, name_); + remotes_.emplace(remote); + } } if (rule.has_downstream_tls_context()) { auto config = rule.downstream_tls_context(); @@ -667,29 +951,41 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } - // inheritpassprecedence bumps up the precedence of a rule in a lower tier to the precedence - // range reserved right after the precedence of the given pass rule. - void inheritPassPrecedence(const PortNetworkPolicyRule& pass_rule) { - precedence_ -= pass_rule.tier_last_precedence_; - precedence_ += pass_rule.precedence_; - } + bool isRemoteWildcard() const { return remotes_.empty() && selectors_.empty(); } - bool isRemoteWildcard() const { return remotes_.empty(); } + bool matchesRemoteId(uint32_t remote_id, const SelectorVersion selector_version) const { + if (isRemoteWildcard()) { + return true; + } + if (!remotes_.empty()) { + return remotes_.contains(remote_id); + } + + for (const auto& selector : selectors_) { + const auto resolved_selector = selector->get(selector_version); + if (resolved_selector && resolved_selector->contains(remote_id)) { + return true; + } + } + return false; + } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, + const SelectorVersion selector_version) const { // proxy_id must match if we have any. if (proxy_id_ && proxy_id != proxy_id_) { return RuleVerdict::None; } // Remote ID must match if we have any. - if (!isRemoteWildcard() && !remotes_.contains(remote_id)) { + if (!matchesRemoteId(remote_id, selector_version)) { return RuleVerdict::None; // no verdict } ASSERT(verdict_ != RuleVerdict::None, "rule must have a verdict"); return verdict_; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { // sni must match if we have any if (!allowed_snis_.empty() && (sni.empty() || std::ranges::none_of(allowed_snis_, [&](const auto& pattern) { @@ -697,13 +993,14 @@ class PortNetworkPolicyRule : public Logger::Loggable { }))) { return RuleVerdict::None; } - return getVerdict(proxy_id, remote_id); + return getVerdict(proxy_id, remote_id, selector_version); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { - auto verdict = getVerdict(proxy_id, remote_id); + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (!hasHttpRules() || verdict != RuleVerdict::Allow) { return verdict; } @@ -727,8 +1024,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { return (header_matched) ? RuleVerdict::Allow : RuleVerdict::None; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - auto verdict = getVerdict(proxy_id, remote_id); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -743,8 +1041,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { // Envoy Metadata matcher, called after deny has already been checked for RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = getVerdict(proxy_id, remote_id); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -795,8 +1094,18 @@ class PortNetworkPolicyRule : public Logger::Loggable { } void toString(int indent, std::string& res) const { - res.append(indent - 2, ' ').append("- remotes: ["); - res.append(fmt::format("{}", fmt::join(remotes_, ","))); + if (!selectors_.empty()) { + res.append(indent - 2, ' ').append("- selectors: ["); + std::vector quoted_selectors; + quoted_selectors.reserve(selectors_.size()); + for (const auto& selector : selectors_) { + quoted_selectors.emplace_back(fmt::format("\"{}\"", selector->name())); + } + res.append(fmt::format("{}", fmt::join(quoted_selectors, ","))); + } else { + res.append(indent - 2, ' ').append("- remotes: ["); + res.append(fmt::format("{}", fmt::join(remotes_, ","))); + } res.append("]\n"); if (!name_.empty()) { @@ -861,9 +1170,10 @@ class PortNetworkPolicyRule : public Logger::Loggable { const RuleVerdict verdict_; const uint16_t proxy_id_; uint32_t precedence_; - uint32_t tier_last_precedence_; + const uint32_t tier_last_precedence_; uint32_t pass_index_; absl::btree_set remotes_; + std::vector selectors_; std::vector allowed_snis_; // All SNIs allowed if empty. std::shared_ptr> @@ -917,13 +1227,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of empty rules. void append(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* selector_resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (const auto& it : rules) { - rules_.emplace_back(std::make_shared(parent, it)); + rules_.emplace_back( + std::make_shared(parent, it, selector_resource_map)); updateFor(rules_.back()); } initialized_ = true; @@ -933,14 +1245,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* selector_resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { rules_.emplace(rules_.begin(), - std::make_shared(parent, it)); + std::make_shared(parent, it, selector_resource_map)); updateFor(rules_.front()); } initialized_ = true; @@ -951,7 +1264,6 @@ class PortNetworkPolicyRules : public Logger::Loggable { // we must add a default allow rule to retain the semantics of the combined rules. void appendRules(const std::vector& rules) { if (initialized_ && rules.empty() != rules_.empty()) { - // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (auto& rule : rules) { @@ -1114,9 +1426,10 @@ class PortNetworkPolicyRules : public Logger::Loggable { RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { auto verdict = forEachRule(can_short_circuit_, [&](const auto& rule) { - return rule.getVerdict(proxy_id, remote_id, headers, log_entry); + return rule.getVerdict(proxy_id, remote_id, headers, log_entry, selector_version); }); ENVOY_LOG(trace, @@ -1125,24 +1438,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return verdict; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }); + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, sni: {}): {}", proxy_id, remote_id, sni, verdict); return verdict; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - return forEachRule( - true, [&](const auto& rule) { return rule.useProxylib(proxy_id, remote_id, l7_proto); }); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + return forEachRule(true, [&](const auto& rule) { + return rule.useProxylib(proxy_id, remote_id, l7_proto, selector_version); + }); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, metadata); }); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, metadata, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, metadata: {}): {}", @@ -1152,20 +1471,24 @@ class PortNetworkPolicyRules : public Logger::Loggable { } RuleVerdict getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getServerTlsContext(tls_ctx, config); }); } RuleVerdict getClientTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getClientTlsContext(tls_ctx, config); }); } @@ -1246,13 +1569,14 @@ const PortNetworkPolicyRules* findPortRules(const PolicySnapshot& map, uint16_t } // namespace -PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port) +PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version) : port_rules_(findPortRules(map, port)), - has_http_rules_(port_rules_ && port_rules_->hasHttpRules()) {} + has_http_rules_(port_rules_ && port_rules_->hasHttpRules()), + selector_version_(selector_version) {} bool PortPolicy::useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { if (port_rules_) { - auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto); + auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto, selector_version_); if (verdict == RuleVerdict::Allow) { return true; } @@ -1272,14 +1596,15 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry, selector_version_) == + RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, sni) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, sni, selector_version_) == RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, @@ -1287,7 +1612,8 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, metadata) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, metadata, selector_version_) == + RuleVerdict::Allow; } Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, @@ -1299,7 +1625,8 @@ Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1314,7 +1641,8 @@ Ssl::ContextSharedPtr PortPolicy::getClientTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1331,7 +1659,8 @@ bool inline rangesOverlap(const PortRange& a, const PortRange& b) { class PortNetworkPolicy : public Logger::Loggable { public: PortNetworkPolicy(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* selector_resource_map) { for (const auto& rule : rules) { // Only TCP supported for HTTP if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { @@ -1490,10 +1819,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules()); + rules.prepend(parent, rule.rules(), selector_resource_map); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules()); + rules.append(parent, rule.rules(), selector_resource_map); } } } else { @@ -1545,7 +1874,9 @@ class PortNetworkPolicy : public Logger::Loggable { } } - const PortPolicy findPortPolicy(uint16_t port) const { return PortPolicy(rules_, port); } + const PortPolicy findPortPolicy(uint16_t port, const SelectorVersion selector_version) const { + return PortPolicy(rules_, port, selector_version); + } void toString(int indent, std::string& res) const { if (rules_.empty()) { @@ -1568,11 +1899,15 @@ class PortNetworkPolicy : public Logger::Loggable { // methods. class PolicyInstanceImpl : public PolicyInstance { public: + friend class NetworkPolicyMapImpl; PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, - const cilium::NetworkPolicy& proto) + const cilium::NetworkPolicy& proto, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMapOverlay* selector_resource_map) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), - parent_(parent), ingress_(parent, policy_proto_.ingress_per_port_policies()), - egress_(parent, policy_proto_.egress_per_port_policies()) {} + parent_(parent), policy_stream_state_(policy_stream_state), + ingress_(parent, policy_proto_.ingress_per_port_policies(), selector_resource_map), + egress_(parent, policy_proto_.egress_per_port_policies(), selector_resource_map) {} bool allowed(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, Envoy::Http::RequestHeaderMap& headers, @@ -1591,7 +1926,9 @@ class PolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t port) const override { - return ingress ? ingress_.findPortPolicy(port) : egress_.findPortPolicy(port); + const auto selector_version = policy_stream_state_->version(); + return ingress ? ingress_.findPortPolicy(port, selector_version) + : egress_.findPortPolicy(port, selector_version); } bool useProxylib(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, @@ -1623,15 +1960,59 @@ class PolicyInstanceImpl : public PolicyInstance { private: const NetworkPolicyMapImpl& parent_; + const PolicyStreamStateConstSharedPtr policy_stream_state_; const PortNetworkPolicy ingress_; const PortNetworkPolicy egress_; }; +void ResourceMap::erasePolicyResource(PolicyMapSnapshot& policy_map, + const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + policy_map.erase(endpoint_ip); + erase(endpoint_ip); + } + erase(resource_name); +} + +void ResourceMapOverlay::erasePolicyResource( + PolicyMapSnapshot& policy_map, const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + policy_map.erase(endpoint_ip); + erase(endpoint_ip); + } + erase(resource_name); +} + +namespace { + +bool policyUsesSelectors(const cilium::NetworkPolicy& policy) { + for (const auto& port_policy : policy.ingress_per_port_policies()) { + if (std::ranges::any_of(port_policy.rules(), + [](const auto& rule) { return rule.selectors_size() > 0; })) { + return true; + } + } + for (const auto& port_policy : policy.egress_per_port_policies()) { + if (std::ranges::any_of(port_policy.rules(), + [](const auto& rule) { return rule.selectors_size() > 0; })) { + return true; + } + } + return false; +} + +} // namespace + // Common base constructor // This is used directly for testing with a file-based subscription -NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe) +NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe, + bool use_delta_xds) : context_(context.serverFactoryContext()) { - impl_ = std::make_unique(context); + impl_ = std::make_shared(context, use_delta_xds); if (subscribe) { impl_->startSubscription(); @@ -1658,9 +2039,11 @@ NetworkPolicyMap::~NetworkPolicyMap() { } bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { - return impl_->getPolicyInstanceImpl(endpoint_policy_name) != nullptr; + return impl_->getPolicyInstanceImpl(endpoint_policy_name); } +bool NetworkPolicyMap::useDeltaXds() const { return impl_->useDeltaXds(); } + void NetworkPolicyMap::startSubscriptionForTest( std::unique_ptr&& subscription) { impl_->startSubscription(std::move(subscription)); @@ -1672,13 +2055,36 @@ Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksFor PolicyStats& NetworkPolicyMap::statsForTest() const { return impl_->stats_; } -NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context) - : context_(context.serverFactoryContext()), map_ptr_(nullptr), +void NetworkPolicyMap::resetStreamForTest() { impl_->resetStreamForTest(); } + +PolicyInstanceConstSharedPtr +NetworkPolicyMap::getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceSharedImpl(endpoint_policy_name); +} + +uint64_t +NetworkPolicyMap::policySelectorStreamGenerationForTest(const PolicyInstance& policy) const { + return impl_->policySelectorStreamGenerationForTestImpl(policy); +} + +SelectorVersion NetworkPolicyMap::policySelectorVersionForTest(const PolicyInstance& policy) const { + return impl_->policySelectorVersionForTestImpl(policy); +} + +NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, + bool use_delta_xds) + : use_delta_xds_(use_delta_xds), context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), init_target_(fmt::format("Cilium Network Policy subscription start"), [this]() { - subscription_->start({}); + if (use_delta_xds_) { + // NPDS always wants all resources, so use an explicit wildcard subscription + // in delta xDS. + subscription_->start({std::string(WildcardResourceName)}); + } else { + subscription_->start({}); + } // Allow listener init to continue before network policy updates are received init_target_.ready(); }), @@ -1717,49 +2123,87 @@ void NetworkPolicyMapImpl::reopenIpcache() { // Cilium agent re-creates IP cache on restart, and the first accepted update on // the new stream must reopen it before workers enforce refreshed identities. IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); - if (ipcache != nullptr) { + if (ipcache) { ENVOY_LOG(info, "Reopening ipcache on new stream"); ipcache->open(); } } -std::shared_ptr -NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, - const PolicyMapSnapshot& old_policy_map) { +std::shared_ptr NetworkPolicyMapImpl::createOrReusePolicy( + const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& old_resource_map, + const ResourceMapOverlay* selector_resource_map) { const uint64_t new_hash = MessageUtil::hash(config); - auto policy_it = old_policy_map.find(config.endpoint_ips()[0]); - if (policy_it != old_policy_map.cend()) { - const auto& old_policy = policy_it->second; + auto it = old_resource_map.find(resource_name); + if (it != old_resource_map.cend()) { + const auto* old_policy_entry = it->second.policyResourceEntry(); + if (old_policy_entry == nullptr) { + return std::make_shared(*this, new_hash, config, + policy_stream_state, selector_resource_map); + } + const auto& old_policy = old_policy_entry->policy; if (old_policy && old_policy->hash_ == new_hash && - Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { + Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config) && + !(selector_resource_map && policyUsesSelectors(config))) { ENVOY_LOG(trace, "New policy is equal to old one, not updating."); return old_policy; } } - // May throw - return std::make_shared(*this, new_hash, config); + return std::make_shared(*this, new_hash, config, policy_stream_state, + selector_resource_map); } -void NetworkPolicyMapImpl::installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, - std::string&& version_name, - uint64_t stream_generation) { +SelectorHandle NetworkPolicyMapImpl::createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, + uint64_t update_version) { + // Compare against the selector visible in the currently prepared update version, not just the + // last published one. Under the single-update-in-flight VersionedMap contract, any selector + // visible in 'update_version' is also the indefinite selector value for that candidate update. + auto selector_value = selector_map_.find(resource_name); + if (selector_value) { + const auto* old_selector = selector_value->get(update_version); + if (old_selector && + old_selector->size() == static_cast(config.remote_identities_size()) && + std::ranges::all_of(config.remote_identities(), [&](const auto remote_identity) { + return old_selector->contains(remote_identity); + })) { + return selector_value; + } + } + + // otherwise create a new one and insert it to the selector map. + + auto selector = new SelectorInstance(); + selector->reserve(config.remote_identities_size()); + for (const auto remote_identity : config.remote_identities()) { + selector->emplace(remote_identity); + } + return selector_map_.insert(resource_name, selector); +} + +void NetworkPolicyMapImpl::installNewPolicyMap( + PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, + std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state) { // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); - const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); + auto new_policy_map_ptr = std::make_unique(std::move(new_policy_map)); + // Publish selector data before publishing the new policy map. New policies created above already + // point at 'policy_stream_state', so any worker that can observe the swapped-in policy map must + // also be able to observe the selector version those policies expect to use. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state->publishVersion(new_version); + } + policy_stream_state_ = policy_stream_state; - // Record stream state only after a successful install. The reserved value 0 - // keeps the initial accepted update on any stream source classified as new. - applied_stream_generation_ = stream_generation; + // old version can be GC'd once all worker threads have quiesced + const auto* old_policy_map = exchange(new_policy_map_ptr.release()); - // Delete the old map once all worker threads have entered their event queues, as this - // is proof that they no longer refer to the old map. - runAfterAllThreads([old_policy_map]() { - // Clean-up in the main thread after all threads have scheduled - delete old_policy_map; - }); + // Delete the old map and first-phase GC old selector versions once all worker threads have + // entered their event queues, as this is proof that they no longer refer to the old map. + scheduleSelectorGCAndDeferredDeletion(new_version, old_policy_map); } // removeInitManager must be called at the end of each policy update @@ -1775,6 +2219,35 @@ void NetworkPolicyMapImpl::removeInitManager() { #endif } +void NetworkPolicyMapImpl::scheduleSelectorDeferredDeletion( + DeferredDeletion&& deferred) { + if (deferred.empty()) { + return; + } + auto deferred_owner = std::make_shared>(std::move(deferred)); + // The callback exists only to keep the deferred-deletion batch alive until all workers have + // quiesced once more. The batch deletes its nodes from the closure destructor. + runAfterAllThreads([deferred_owner]() {}); +} + +void NetworkPolicyMapImpl::scheduleSelectorGCAndDeferredDeletion( + uint64_t published_version, const PolicyMapSnapshot* old_policy_map) { + if (published_version == 0 && old_policy_map == nullptr) { + return; + } + runAfterAllThreads([shared_this = shared_from_this(), published_version, old_policy_map]() { + // Clean-up in the main thread after all worker threads have scheduled. + // Delete the old policy map before selector GC. Old policies are the only remaining users of + // old-stream selector versions; once the old map is gone after this quiescence point, those + // selector versions may be unlinked and deferred for deletion. + delete old_policy_map; + if (published_version == 0) { + return; + } + shared_this->scheduleSelectorDeferredDeletion(shared_this->selector_map_.gc(published_version)); + }); +} + // onConfigUpdate parses the new network policy resources, allocates a new policy map and atomically // swaps it in place of the old policy map. Throws if any of the 'resources' can not be // parsed. Otherwise an OK status is returned without pausing NPDS gRPC stream, causing a new @@ -1783,7 +2256,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { auto stream_generation = streamGeneration(); - const bool is_new_stream = stream_generation != applied_stream_generation_; + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", instance_id_, resources.size(), version_info); stats_.updates_total_.inc(); @@ -1805,11 +2278,18 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // SDS secrets will use this! transport_factory_context_->setInitManager(version_init_manager); - const auto* old_policy_map = load(); + const auto& old_resource_map = resource_map_; + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; PolicyMapSnapshot new_policy_map; + std::vector> resource_entries; try { for (const auto& resource : resources) { const auto& config = dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceNameHasNoWhitespace(resource_name, "Network Policy resource name"); if (config.endpoint_ips().empty()) { throw EnvoyException("Network Policy has no endpoint ips"); } @@ -1818,11 +2298,16 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( "version {}", config.endpoint_id(), config.endpoint_ips()[0], version_info); - auto policy = createOrReusePolicy(config, *old_policy_map); + auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, + old_resource_map, nullptr); + if (!resource_name.empty()) { + resource_entries.emplace_back(resource_name, ResourceKey::policyResource(policy)); + } for (const auto& endpoint_ip : config.endpoint_ips()) { ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", endpoint_ip); // new_policy_map is not exception safe, policy must be computed separately! - new_policy_map.emplace(endpoint_ip, policy); + new_policy_map.insert_or_assign(endpoint_ip, policy); + resource_entries.emplace_back(endpoint_ip, ResourceKey::policyEndpointIp()); } } } catch (const EnvoyException& e) { @@ -1834,7 +2319,290 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( removeInitManager(); installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), - stream_generation); + policy_stream_state); + resource_map_.replaceWith(std::move(resource_entries)); + + return absl::OkStatus(); +} + +absl::Status NetworkPolicyMapImpl::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + auto stream_generation = streamGeneration(); + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + const auto& old_resource_map = resource_map_; + bool updates_policies = false; + bool updates_selectors = false; + for (const auto& removed_resource : removed_resources) { + validateResourceNameHasNoWhitespace(removed_resource, + "Network Policy delta removed resource name"); + auto resource_it = old_resource_map.find(removed_resource); + if (resource_it == old_resource_map.end()) { + continue; + } + if (resource_it->second.selectorResourceEntry()) { + updates_selectors = true; + } else { + updates_policies = true; + } + } + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + if (resource_name.empty()) { + throw EnvoyException("Network Policy delta resource has no name"); + } + validateResourceNameHasNoWhitespace(resource_name, "Network Policy delta resource name"); + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + updates_policies = true; + break; + case cilium::NetworkPolicyResource::kSelector: + updates_selectors = true; + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + break; + } + } + ENVOY_LOG(debug, + "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " + "version: {}, updates_selectors: {}, updates_policies: {}", + instance_id_, added_resources.size(), removed_resources.size(), system_version_info, + updates_selectors, updates_policies); + stats_.updates_total_.inc(); + + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy stream"); + reopenIpcache(); + } + removeInitManager(); + + if (!is_new_stream && updates_selectors && !updates_policies) { + ResourceMapOverlay pending_resource_map(resource_map_); + + try { + const auto selector_update_version = selector_map_.prepareNextVersion(); + + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing network policy resource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException(fmt::format( + "Network Policy delta removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } + if (resource_entry->policyResourceEntry()) { + throw EnvoyException( + fmt::format("Network Policy delta removed resource '{}' refers to a policy resource", + removed_resource)); + } + selector_map_.clear(removed_resource); + pending_resource_map.erase(removed_resource); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + pending_resource_map.eraseSelectorResourceIfPresent(resource_name); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: { + ENVOY_LOG(debug, + "Received delta Network Policy selector resource {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto selector_handle = createOrReuseSelector(resource_name, typed_resource.selector(), + selector_update_version); + if (!pending_resource_map.emplace(resource_name, + ResourceKey::selectorResource(selector_handle))) { + throw EnvoyException(fmt::format( + "Network Policy delta update has duplicate resource key '{}'", resource_name)); + } + break; + } + case cilium::NetworkPolicyResource::kPolicy: + IS_ENVOY_BUG("Selector-only delta Network Policy update unexpectedly included a policy"); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("Network Policy delta resource has no payload"); + } + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy delta update for version {} failed: {}", system_version_info, + e.what()); + stats_.updates_rejected_.inc(); + scheduleSelectorDeferredDeletion(selector_map_.revert()); + throw; // re-throw + } + + // Same-stream selector-only updates become visible to existing policies by first publishing the + // selector version itself and only then publishing that version number through the shared + // stream state. Reversing this order would let workers observe a selector version that has not + // yet been published in the selector map. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state_->publishVersion(new_version); + scheduleSelectorGCAndDeferredDeletion(new_version); + } + std::move(pending_resource_map).applyTo(resource_map_); + return absl::OkStatus(); + } + + std::string version_name = fmt::format("NetworkPolicyMap version {}", system_version_info); + Init::ManagerImpl version_init_manager(version_name); + transport_factory_context_->setInitManager(version_init_manager); + + const auto* old_policy_map = load(); + PolicyMapSnapshot new_policy_map = is_new_stream ? PolicyMapSnapshot{} : *old_policy_map; + ResourceMapOverlay pending_resource_map = + is_new_stream ? ResourceMapOverlay() : ResourceMapOverlay(resource_map_); + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; + try { + const auto selector_update_version = selector_map_.prepareNextVersion(); + + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing network policy resource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (resource_entry->selectorResourceEntry()) { + selector_map_.clear(removed_resource); + pending_resource_map.erase(removed_resource); + continue; + } + if (pending_resource_map.erasePolicyResourceIfPresent(new_policy_map, removed_resource)) { + continue; + } + throw EnvoyException( + fmt::format("Network Policy delta removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + const auto* resource_entry = pending_resource_map.findEntry(resource_name); + if (resource_entry == nullptr) { + continue; + } + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: + pending_resource_map.eraseSelectorResourceIfPresent(resource_name); + break; + case cilium::NetworkPolicyResource::kPolicy: + pending_resource_map.erasePolicyResourceIfPresent(new_policy_map, resource_name); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + break; + } + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + + ENVOY_LOG(debug, + "Received delta Network Policy selector resource {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto selector_handle = + createOrReuseSelector(resource_name, typed_resource.selector(), selector_update_version); + if (!pending_resource_map.emplace(resource_name, + ResourceKey::selectorResource(selector_handle))) { + throw EnvoyException(fmt::format( + "Network Policy delta update has duplicate resource key '{}'", resource_name)); + } + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: + break; + case cilium::NetworkPolicyResource::kPolicy: { + const auto& config = typed_resource.policy(); + if (config.endpoint_ips().empty()) { + throw EnvoyException("Network Policy has no endpoint ips"); + } + if (config.endpoint_id() == 0) { + throw EnvoyException("Network Policy endpoint_id must be non-zero"); + } + ENVOY_LOG(debug, + "Received delta Network Policy resource {} for endpoint {}, endpoint_ip {} in " + "onConfigUpdate() version {}", + resource_name, config.endpoint_id(), config.endpoint_ips()[0], + system_version_info); + + auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, + old_resource_map, &pending_resource_map); + if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { + throw EnvoyException(fmt::format( + "Network Policy delta update has duplicate resource key '{}'", resource_name)); + } + for (const auto& endpoint_ip : config.endpoint_ips()) { + ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); + if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp())) { + throw EnvoyException(fmt::format( + "Network Policy delta update has duplicate resource key '{}'", endpoint_ip)); + } + if (!new_policy_map.emplace(endpoint_ip, policy).second) { + throw EnvoyException(fmt::format( + "Network Policy delta update has duplicate resource key '{}'", endpoint_ip)); + } + } + break; + } + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("Network Policy delta resource has no payload"); + } + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy delta update for version {} failed: {}", system_version_info, + e.what()); + stats_.updates_rejected_.inc(); + removeInitManager(); + scheduleSelectorDeferredDeletion(selector_map_.revert()); + throw; // re-throw + } + removeInitManager(); + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), + policy_stream_state); + std::move(pending_resource_map).applyTo(resource_map_); return absl::OkStatus(); } @@ -1887,7 +2655,7 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t) const override { - return ingress ? PortPolicy(empty_map_, 0) : PortPolicy(empty_map_, 1); + return ingress ? PortPolicy(empty_map_, 0, versionMin) : PortPolicy(empty_map_, 1, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -1930,7 +2698,7 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool, uint16_t) const override { - return PortPolicy(empty_map_, 0); + return PortPolicy(empty_map_, 0, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -1968,6 +2736,32 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons return nullptr; } +PolicyInstanceConstSharedPtr +NetworkPolicyMapImpl::getPolicyInstanceSharedImpl(const std::string& endpoint_ip) const { + const auto* map = load(); + auto it = map->find(endpoint_ip); + if (it != map->end()) { + return it->second; + } + return nullptr; +} + +uint64_t NetworkPolicyMapImpl::policySelectorStreamGenerationForTestImpl( + const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->streamGeneration(); + } + return 0; +} + +SelectorVersion +NetworkPolicyMapImpl::policySelectorVersionForTestImpl(const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->version(); + } + return versionMin; +} + // getPolicyInstance return a const reference to a policy in the policy map for the given // 'endpoint_ip'. If there is no policy for the given IP, a default policy is returned, // controlled by the 'default_allow_egress' argument as follows: @@ -1982,9 +2776,7 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons const PolicyInstance& NetworkPolicyMap::getPolicyInstance(const std::string& endpoint_ip, bool default_allow_egress) const { const auto* policy = impl_->getPolicyInstanceImpl(endpoint_ip); - return policy != nullptr ? *policy - : default_allow_egress ? getAllowAllEgressPolicy() - : getDenyAllPolicy(); + return policy ? *policy : default_allow_egress ? getAllowAllEgressPolicy() : getDenyAllPolicy(); } } // namespace Cilium diff --git a/cilium/network_policy.h b/cilium/network_policy.h index 69d7c7558..cd4b9900f 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -23,6 +23,7 @@ #include "source/common/common/logger.h" #include "source/common/common/macros.h" #include "source/common/common/thread.h" +#include "source/common/config/opaque_resource_decoder_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" @@ -39,6 +40,7 @@ namespace Cilium { class PortNetworkPolicyRules; class PolicySnapshot; +using SelectorVersion = uint64_t; // PortPolicy holds a reference to a set of rules in a policy map that apply to the given port. // Methods then iterate through the set to determine if policy allows or denies. This is needed to @@ -49,7 +51,7 @@ class PortPolicy : public Logger::Loggable { friend class PortNetworkPolicy; friend class DenyAllPolicyInstanceImpl; friend class AllowAllEgressPolicyInstanceImpl; - PortPolicy(const PolicySnapshot& map, uint16_t port); + PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version); public: // If hasHttpRules() returns false, then HTTP policy enforcement can be skipped, @@ -95,6 +97,7 @@ class PortPolicy : public Logger::Loggable { // rules. const PortNetworkPolicyRules* port_rules_; const bool has_http_rules_; + const SelectorVersion selector_version_; }; class IpAddressPair { @@ -166,6 +169,9 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { ProtobufMessage::ValidationVisitor& validation_visitor_; }; +using NetworkPolicyResourceDecoder = + Envoy::Config::OpaqueResourceDecoderImpl; + /** * All Cilium L7 filter stats. @see stats_macros.h */ @@ -187,10 +193,12 @@ class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: - NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe = false); + NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe = false, + bool use_delta_xds = false); ~NetworkPolicyMap() override; bool exists(const std::string& endpoint_policy_name) const; + bool useDeltaXds() const; const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; @@ -202,12 +210,17 @@ class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable&& subscription); Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: Server::Configuration::ServerFactoryContext& context_; - std::unique_ptr impl_; + std::shared_ptr impl_; }; using NetworkPolicyMapSharedPtr = std::shared_ptr; diff --git a/cilium/versioned.h b/cilium/versioned.h new file mode 100644 index 000000000..9ba0bfa40 --- /dev/null +++ b/cilium/versioned.h @@ -0,0 +1,518 @@ +#pragma once + +// NOLINT(namespace-envoy) + +#include +#include +#include +#include +#include +#include + +#include "source/common/common/assert.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +constexpr uint64_t versionMin = 0; +constexpr uint64_t versionMax = std::numeric_limits::max() - 1; +constexpr uint64_t versionNotRemoved = std::numeric_limits::max(); + +// Versioned.h provides a lock-free reader / single-writer versioned-value container, +// based on earlier implementations in OVS (C) and Cilium (Go). +// +// API contract and intended use: +// +// 1. Object model +// - T is a CRTP node type inheriting from VersionedNode. +// - VersionedValue stores a linked list of historical T nodes, newest first. +// - VersionedReadable exposes only the worker-safe read API: get(version). +// - VersionedHandle is a shared_ptr, so handles intentionally +// expose only the readable API to worker/runtime code. Main-thread mutation goes through +// VersionedMap, which keeps mutable shared_ptr internally. +// - The optional readable base R allows a handle to expose extra read-only metadata (for +// example selector names) without exposing VersionedValue mutators. +// +// 2. Threading model +// - There is exactly one mutator thread: the main thread. +// - Worker threads may concurrently call VersionedReadable::get(version) and may traverse nodes +// reached from a handle without additional locking. +// - Any future cross-thread mutation must still preserve the exclusive-writer rule, e.g. by +// holding an exclusive lock around all mutation. +// - VersionedMap itself is main-thread-only. Its name directory (map_), dirty set, and +// transactional state must not be accessed from worker threads. +// +// 3. VersionedMap transaction flow +// - prepareNextVersion() starts a new unpublished update and returns the candidate version to +// use for any main-thread lookups against in-flight state. +// - insert(key, value) reuses the existing stable handle if the key is still present in the +// main-thread directory; otherwise it creates a fresh handle. +// - clear(key) marks the key's current value invisible in the candidate version but does not +// erase the key from the main-thread directory immediately. +// - publishNextVersion() publishes all pending changes, removes keys whose handle has no visible +// value in the published version, and returns the new published version, or 0 if there was +// nothing to publish. +// - revert() discards the unpublished update, restores visibility of older nodes as needed, and +// removes keys whose handle has no value in the published version. +// - find(key) returns the stable readable handle parked under the key in the main-thread +// directory. It does not itself answer whether the key is visible in any specific version; use +// handle->get(version) for that. +// +// 4. Stable-handle semantics +// - Updating an active key writes a new version onto the same stable handle. +// - clear() followed by re-add of the same key in the same unpublished update also reuses that +// same handle; the caller should compare against the candidate version returned from +// prepareNextVersion(), not against the currently published version. +// - Once a key has been fully removed across a published version boundary, the name disappears +// from VersionedMap. A later same-name add creates a fresh stable handle. +// - This is why old published policies can keep using an old cleared handle while refreshed +// policies bind a new one after a later same-name re-add. +// +// 5. Reader semantics +// - get(version) walks from the current head and returns the first node visible in the requested +// version. +// - The walk stops at the first node that was added before the requested version but is no +// longer visible there; all remaining nodes are older and therefore already invisible in that +// version. +// - Querying an older published version after newer publishes is logically stale and may produce +// outdated results, but it must remain memory-safe. +// +// 6. Deferred deletion / grace periods +// - Unpublished nodes removed by revert() were never visible to workers, but are still unlinked +// into a DeferredDeletion batch so that any concurrent traversal that had already stepped into +// them cannot race with delete. +// - gc(published_version) is only the first GC phase. It must be called only after all workers +// have quiesced for that published version, unlinks nodes that are no longer visible to +// anyone, and returns them in a DeferredDeletion batch. +// - The returned DeferredDeletion batch must stay alive until all workers have quiesced once +// more. Destroying the batch performs the actual node deletion. +// - The production pattern is therefore: publish version N, wait for worker quiescence, call +// gc(N), then keep the returned DeferredDeletion alive for one more quiescence round. +// +// 7. High-level memory-model rationale +// - The main thread publishes new list links with 'release' stores to the list node pointers. +// - Workers traverse with 'acquire' loads from the list node pointers, so they either observe +// the old reachable chain or the newly published/unlinked chain, but not torn pointer state. +// - published_version_ is stored with 'release' in publishNextVersion() and loaded with +// 'acquire' by readers, so readers that observe a new published version also observe the +// prior main-thread mutations that made that version reachable. +// - remove_version_ uses 'relaxed' atomic loads/stores because it controls logical visibility, +// not structural reachability. Structural coherence and memory safety come from the +// 'release'/'acquire' ordering on list node pointers plus the extra deferred-deletion grace +// period. +// - Many mutation-side loads/stores are 'relaxed' because mutation is single-threaded on the +// main thread. +// +// 8. DeferredDeletion implementation detail +// - DeferredDeletion reuses VersionedNode::next_ to chain together unlinked nodes awaiting +// destruction. This keeps deferred deletion self-contained without an extra node container. +// - Public code must not rely on any semantics of detached invisible tails beyond memory safety; +// only get(version) is the supported read API. +// +// 9. Test coverage in tests/versioned_test.cc +// - Value-level tests cover visibility rules, shadowing, clear(), revert(), first-phase GC, and +// deferred deletion ownership/move semantics. +// - Map-level tests cover version numbering, publish-without-change, insert/update/clear/revert, +// same-handle reuse for active updates and same-update clear+read, fresh-handle behavior +// after published removal, and dirty-handle retention across multiple future GC runs. +// - Stable validator tests verify consistent published snapshots and post-unlink/pre-deletion +// states. +// - A multithreaded chaos test exercises one writer with multiple busy-loop readers, repeated +// publish/revert/GC cycles, stale-version reads, and traversal safety under deferred deletion. +// +template class VersionedReadable; +template > class VersionedValue; +template class DeferredDeletion; +template > struct VersionedTestAccess; + +// VersionedNode is CRTP type on T, T must inherit from VersionedNode +template class VersionedNode { +public: + template + VersionedNode() + : next_(nullptr), add_version_(versionNotRemoved), remove_version_(versionNotRemoved) {} + +private: + template friend class VersionedReadable; + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + bool isUnpublished() const { return add_version_ == versionNotRemoved; } + + void setAddVersion(uint64_t version) { + ASSERT(version < versionNotRemoved); + add_version_ = version; // set before published + } + + void removeInVersion(uint64_t version) { + remove_version_.store(version, std::memory_order_relaxed); + } + + bool isVisibleInVersion(uint64_t version) const { + return add_version_ <= version && version < remove_version_.load(std::memory_order_relaxed); + } + + bool isAddedAfterVersion(uint64_t version) const { return add_version_ > version; } + + bool isAddedBeforeVersion(uint64_t version) const { return add_version_ < version; } + + bool isEventuallyInvisible() const { + return remove_version_.load(std::memory_order_relaxed) < versionNotRemoved; + } + + bool isVisibleOnlyBeforeVersion(uint64_t version) const { + return remove_version_.load(std::memory_order_relaxed) <= version; + } + + bool isRemovedAfterVersion(uint64_t version) const { + const auto remove_version = remove_version_.load(std::memory_order_relaxed); + return remove_version < versionNotRemoved && version < remove_version; + } + + VersionedNode* getNext() const { return next_.load(std::memory_order_acquire); } + + void setNext(VersionedNode* next) { return next_.store(next, std::memory_order_release); } + + VersionedNode* getNextProtected() const { return next_.load(std::memory_order_relaxed); } + std::atomic*>* getNextP() { return &next_; } + + T* value() { return static_cast(this); } + const T* value() const { return static_cast(this); } + + std::atomic*> next_; + + uint64_t add_version_; // Version object was added in. + std::atomic remove_version_; // Version object is removed in. +}; + +template class VersionedReadable { +public: + const T* get(uint64_t version) const { + for (auto node = head_.load(std::memory_order_acquire); node; node = node->getNext()) { + if (node->isVisibleInVersion(version)) { + return node->value(); + } + // stop traveral on first non-visible version that was added before this version, all + // remaining nodes are already invisible. + if (node->isAddedBeforeVersion(version)) { + break; + } + } + return nullptr; // not found + } + +protected: + VersionedReadable() : head_(nullptr) {} + + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + std::atomic*> head_; +}; + +// T is the CRTP node type stored in the version chain. R is the read-only interface exposed +// through VersionedHandle: it defaults to VersionedReadable, but callers may provide a richer +// readable base when handles need extra read-side API without exposing the main-thread mutators. +template class VersionedValue : public R { +public: + using Readable = R; + using R::head_; + + VersionedValue() = default; + + template >> + explicit VersionedValue(Args&&... args) : R(std::forward(args)...) {} + + ~VersionedValue() { + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + delete node->value(); + } + } + + // make all versions invisible starting at "version" + void clear(uint64_t version) { + for (auto node = head_.load(std::memory_order_relaxed); node; node = node->getNextProtected()) { + if (!node->isEventuallyInvisible()) { + node->removeInVersion(version); + } + } + } + + void set(uint64_t version, VersionedNode* node) { + // publish a new version that is visible starting at "version" + ASSERT(node->isUnpublished()); // node not pushed into a list yet + node->setAddVersion(version); + node->setNext(head_.load(std::memory_order_relaxed)); + head_.store(node, std::memory_order_release); + + // make all other versions invisible at "version", starting from the first old node. + for (node = node->getNextProtected(); node; node = node->getNextProtected()) { + if (node->isVisibleInVersion(version)) { + node->removeInVersion(version); + } + } + } + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + + // Unlink unpublished versions added after 'version' and append them to 'deferred' for deletion + // after an additional grace period. Removed nodes that were still visible in 'version' are + // restored. + void revert(uint64_t version, DeferredDeletion& deferred) { + auto prev = &head_; + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isAddedAfterVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isRemovedAfterVersion(version)) { + // visibility was limited, restore + node->removeInVersion(versionNotRemoved); + } + prev = node->getNextP(); + } + } + + // Unlink all versions not visible to anyone after the "version" was published and all readers + // have quiesced, but do not delete them yet. Unlinked nodes are appended to 'deferred' and are + // only deleted after one more worker-thread quiescence round. Any nodes not yet visible at + // "version" must not be unlinked. + // Returns true if there are no future version removals left in this handle after this first + // phase GC run. + bool gcForVersion(uint64_t version, DeferredDeletion& deferred) { + ASSERT(version < versionNotRemoved); + auto prev = &head_; + VersionedNode* next; + bool has_future_gc_work = false; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isVisibleOnlyBeforeVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isEventuallyInvisible()) { + has_future_gc_work = true; + } + prev = node->getNextP(); + } + return !has_future_gc_work; + } + +protected: + friend class DeferredDeletion; + template friend struct VersionedTestAccess; +}; + +template class DeferredDeletion : protected VersionedValue { +public: + // Bring `head_` from the templated base class into this scope so the rest of the code can use + // normal unqualified member access. Without this, dependent-base lookup would require + // `this->head_` everywhere. + using VersionedValue::head_; + + DeferredDeletion() = default; + DeferredDeletion(const DeferredDeletion&) = delete; + DeferredDeletion& operator=(const DeferredDeletion&) = delete; + DeferredDeletion(DeferredDeletion&& other) noexcept { + head_.store(other.head_.exchange(nullptr, std::memory_order_relaxed), + std::memory_order_relaxed); + tail_ = std::exchange(other.tail_, nullptr); + } + DeferredDeletion& operator=(DeferredDeletion&&) noexcept = delete; + ~DeferredDeletion() = default; + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + +private: + template friend class VersionedValue; + + void push(VersionedNode* node) { + node->setNext(nullptr); + if (tail_) { + tail_->setNext(node); + } else { + // Only the main thread is accessing the deferred-deletion head, hence relaxed. + head_.store(node, std::memory_order_relaxed); + } + tail_ = node; + } + + VersionedNode* tail_{nullptr}; +}; + +template > +using VersionedHandle = std::shared_ptr; + +// VersionedMap is main-thread / exclusive-writer only: map_, pending_keys_, dirty_values_, and +// all mutation methods are accessed only by the single publishing thread. Worker threads only use +// the published Handle objects and then call the read-only handle API. +template > class VersionedMap { +public: + static_assert(std::is_base_of_v); + static_assert(std::is_base_of_v, typename V::Readable>); + using Handle = VersionedHandle; + + uint64_t getVersion() const { return published_version_.load(std::memory_order_acquire); } + + Handle find(const K& key) const { + auto it = map_.find(key); + if (it != map_.cend()) { + return it->second; + } + return nullptr; + } + + uint64_t prepareNextVersion() { + ASSERT(pending_keys_.empty()); + return ++next_version_; + } + + Handle insert(const K& key, T* value) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto [it, inserted] = map_.try_emplace(key); + if (inserted) { + if constexpr (std::is_constructible_v) { + it->second = std::make_shared(key); + } else { + it->second = std::make_shared(); + } + } + auto& handle = it->second; + handle->set(next_version_, value); + pending_keys_.emplace(key); + return handle; + } + + void clear(const K& key) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto it = map_.find(key); + if (it != map_.cend()) { + it->second->clear(next_version_); + pending_keys_.emplace(key); + } + } + + // returns the newly published version, or 0 if there was nothing to publish + uint64_t publishNextVersion() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return 0; + } + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + dirty_values_.emplace(it->second); + if (it->second->get(next_version_) == nullptr) { + map_.erase(it); + } + } + published_version_.store(next_version_, std::memory_order_release); + pending_keys_.clear(); + return next_version_; + } + + DeferredDeletion revert() { + auto version = published_version_.load(std::memory_order_relaxed); + DeferredDeletion deferred; + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + it->second->revert(version, deferred); + if (it->second->get(version) == nullptr) { + map_.erase(it); + } + } + pending_keys_.clear(); + next_version_ = version; + return deferred; + } + + // First phase GC after all readers have quiesced for 'published_version'. This unlinks nodes that + // are no longer visible and returns them in a deferred-deletion batch that must survive until all + // readers have quiesced once more before it is destroyed. + DeferredDeletion gc(uint64_t published_version) { + DeferredDeletion deferred; + for (auto it = dirty_values_.begin(); it != dirty_values_.end();) { + const auto& handle = *it; + if (handle->gcForVersion(published_version, deferred)) { + dirty_values_.erase(it++); + continue; + } + ++it; + } + return deferred; + } + +private: + friend struct VersionedTestAccess; + using MutableHandle = std::shared_ptr; + + // Persistent published state. + std::atomic published_version_{versionMin}; + absl::flat_hash_map map_; + absl::flat_hash_set dirty_values_; + + // Transactional state for the currently prepared unpublished update. + // Valid only after prepareNextVersion() and until publishNextVersion() or revert(). + // `next_version_` is the candidate version being built, and `pending_keys_` + // contains the keys touched in that candidate update. The container is reused + // across updates to avoid repeated allocation churn. + uint64_t next_version_{versionMin}; + absl::flat_hash_set pending_keys_; +}; + +template struct VersionedTestAccess { + using Map = VersionedMap; + using Handle = typename Map::Handle; + using MutableHandle = typename Map::MutableHandle; + using Node = VersionedNode; + + static const absl::flat_hash_map& entries(const Map& map) { return map.map_; } + + static const absl::flat_hash_set& dirtyValues(const Map& map) { + return map.dirty_values_; + } + + static const Node* head(const Handle& handle) { + if (handle == nullptr) { + return nullptr; + } + auto value = std::static_pointer_cast(handle); + return value->head_.load(std::memory_order_acquire); + } + + static const Node* next(const Node* node) { + return node ? node->next_.load(std::memory_order_acquire) : nullptr; + } + + static const T* value(const Node* node) { return node ? node->value() : nullptr; } + + static uint64_t addVersion(const Node* node) { return node->add_version_; } + + static uint64_t removeVersion(const Node* node) { + return node->remove_version_.load(std::memory_order_relaxed); + } + + static bool isVisibleInVersion(const Node* node, uint64_t version) { + return node && node->isVisibleInVersion(version); + } + + static bool isAddedAfterVersion(const Node* node, uint64_t version) { + return node && node->isAddedAfterVersion(version); + } +}; diff --git a/go/cilium/api/bpf_metadata.pb.go b/go/cilium/api/bpf_metadata.pb.go index 3b860b778..185d778db 100644 --- a/go/cilium/api/bpf_metadata.pb.go +++ b/go/cilium/api/bpf_metadata.pb.go @@ -7,6 +7,7 @@ package cilium import ( + v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" _ "github.com/envoyproxy/protoc-gen-validate/validate" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -86,8 +87,15 @@ type BpfMetadata struct { CacheEntryTtl *durationpb.Duration `protobuf:"bytes,14,opt,name=cache_entry_ttl,json=cacheEntryTtl,proto3" json:"cache_entry_ttl,omitempty"` // Cache is garbage collected at interval 10 times the ttl (default 30 ms). CacheGcInterval *durationpb.Duration `protobuf:"bytes,15,opt,name=cache_gc_interval,json=cacheGcInterval,proto3" json:"cache_gc_interval,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Configuration for the source of NPDS updates. Currently this field is not supported. + NpdsConfig *v3.ConfigSource `protobuf:"bytes,16,opt,name=npds_config,json=npdsConfig,proto3" json:"npds_config,omitempty"` + // Use delta NPDS rather than the state-of-the-world protocol. + // Even with delta NPDS, each new stream starts with a full dump. + // Only wildcard subscriptions are supported. + // All listeners on the node must agree on this setting. + UseDeltaNpds bool `protobuf:"varint,17,opt,name=use_delta_npds,json=useDeltaNpds,proto3" json:"use_delta_npds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BpfMetadata) Reset() { @@ -225,11 +233,25 @@ func (x *BpfMetadata) GetCacheGcInterval() *durationpb.Duration { return nil } +func (x *BpfMetadata) GetNpdsConfig() *v3.ConfigSource { + if x != nil { + return x.NpdsConfig + } + return nil +} + +func (x *BpfMetadata) GetUseDeltaNpds() bool { + if x != nil { + return x.UseDeltaNpds + } + return false +} + var File_cilium_api_bpf_metadata_proto protoreflect.FileDescriptor const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\n" + - "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\x94\x06\n" + + "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xff\x06\n" + "\vBpfMetadata\x12\x19\n" + "\bbpf_root\x18\x01 \x01(\tR\abpfRoot\x12\x1d\n" + "\n" + @@ -247,7 +269,10 @@ const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\fipcache_name\x18\f \x01(\tR\vipcacheName\x12\x1b\n" + "\tuse_nphds\x18\r \x01(\bR\buseNphds\x12A\n" + "\x0fcache_entry_ttl\x18\x0e \x01(\v2\x19.google.protobuf.DurationR\rcacheEntryTtl\x12E\n" + - "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcIntervalB!\n" + + "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcInterval\x12C\n" + + "\vnpds_config\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\n" + + "npdsConfig\x12$\n" + + "\x0euse_delta_npds\x18\x11 \x01(\bR\fuseDeltaNpdsB!\n" + "\x1f_original_source_so_linger_timeB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -266,16 +291,18 @@ var file_cilium_api_bpf_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, var file_cilium_api_bpf_metadata_proto_goTypes = []any{ (*BpfMetadata)(nil), // 0: cilium.BpfMetadata (*durationpb.Duration)(nil), // 1: google.protobuf.Duration + (*v3.ConfigSource)(nil), // 2: envoy.config.core.v3.ConfigSource } var file_cilium_api_bpf_metadata_proto_depIdxs = []int32{ 1, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration 1, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration 1, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration - 3, // [3:3] is the sub-list for method output_type - 3, // [3:3] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // 3: cilium.BpfMetadata.npds_config:type_name -> envoy.config.core.v3.ConfigSource + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_cilium_api_bpf_metadata_proto_init() } diff --git a/go/cilium/api/bpf_metadata.pb.validate.go b/go/cilium/api/bpf_metadata.pb.validate.go index 8180998d2..64fbc69cc 100644 --- a/go/cilium/api/bpf_metadata.pb.validate.go +++ b/go/cilium/api/bpf_metadata.pb.validate.go @@ -175,6 +175,37 @@ func (m *BpfMetadata) validate(all bool) error { } } + if all { + switch v := interface{}(m.GetNpdsConfig()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, BpfMetadataValidationError{ + field: "NpdsConfig", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, BpfMetadataValidationError{ + field: "NpdsConfig", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetNpdsConfig()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return BpfMetadataValidationError{ + field: "NpdsConfig", + reason: "embedded message failed validation", + cause: err, + } + } + } + + // no validation rules for UseDeltaNpds + if m.OriginalSourceSoLingerTime != nil { // no validation rules for OriginalSourceSoLingerTime } diff --git a/go/cilium/api/npds.pb.go b/go/cilium/api/npds.pb.go index d708a6827..917e190d2 100644 --- a/go/cilium/api/npds.pb.go +++ b/go/cilium/api/npds.pb.go @@ -75,7 +75,7 @@ func (x HeaderMatch_MatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MatchAction.Descriptor instead. func (HeaderMatch_MatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 0} } type HeaderMatch_MismatchAction int32 @@ -130,7 +130,139 @@ func (x HeaderMatch_MismatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MismatchAction.Descriptor instead. func (HeaderMatch_MismatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 1} +} + +// A delta NPDS resource that carries either an endpoint policy or a shared selector. +type NetworkPolicyResource struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Resource: + // + // *NetworkPolicyResource_Policy + // *NetworkPolicyResource_Selector + Resource isNetworkPolicyResource_Resource `protobuf_oneof:"resource"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkPolicyResource) Reset() { + *x = NetworkPolicyResource{} + mi := &file_cilium_api_npds_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkPolicyResource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkPolicyResource) ProtoMessage() {} + +func (x *NetworkPolicyResource) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkPolicyResource.ProtoReflect.Descriptor instead. +func (*NetworkPolicyResource) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} +} + +func (x *NetworkPolicyResource) GetResource() isNetworkPolicyResource_Resource { + if x != nil { + return x.Resource + } + return nil +} + +func (x *NetworkPolicyResource) GetPolicy() *NetworkPolicy { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Policy); ok { + return x.Policy + } + } + return nil +} + +func (x *NetworkPolicyResource) GetSelector() *Selector { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Selector); ok { + return x.Selector + } + } + return nil +} + +type isNetworkPolicyResource_Resource interface { + isNetworkPolicyResource_Resource() +} + +type NetworkPolicyResource_Policy struct { + Policy *NetworkPolicy `protobuf:"bytes,1,opt,name=policy,proto3,oneof"` +} + +type NetworkPolicyResource_Selector struct { + Selector *Selector `protobuf:"bytes,2,opt,name=selector,proto3,oneof"` +} + +func (*NetworkPolicyResource_Policy) isNetworkPolicyResource_Resource() {} + +func (*NetworkPolicyResource_Selector) isNetworkPolicyResource_Resource() {} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + RemoteIdentities []uint32 `protobuf:"varint,1,rep,packed,name=remote_identities,json=remoteIdentities,proto3" json:"remote_identities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_cilium_api_npds_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} +} + +func (x *Selector) GetRemoteIdentities() []uint32 { + if x != nil { + return x.RemoteIdentities + } + return nil } // A network policy that is enforced by a filter on the network flows to/from @@ -161,7 +293,7 @@ type NetworkPolicy struct { func (x *NetworkPolicy) Reset() { *x = NetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -173,7 +305,7 @@ func (x *NetworkPolicy) String() string { func (*NetworkPolicy) ProtoMessage() {} func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -186,7 +318,7 @@ func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPolicy.ProtoReflect.Descriptor instead. func (*NetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} } func (x *NetworkPolicy) GetEndpointIps() []string { @@ -240,7 +372,7 @@ type PortNetworkPolicy struct { func (x *PortNetworkPolicy) Reset() { *x = PortNetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +384,7 @@ func (x *PortNetworkPolicy) String() string { func (*PortNetworkPolicy) ProtoMessage() {} func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +397,7 @@ func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicy.ProtoReflect.Descriptor instead. func (*PortNetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} } func (x *PortNetworkPolicy) GetPort() uint32 { @@ -328,7 +460,7 @@ type TLSContext struct { func (x *TLSContext) Reset() { *x = TLSContext{} - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -340,7 +472,7 @@ func (x *TLSContext) String() string { func (*TLSContext) ProtoMessage() {} func (x *TLSContext) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -353,7 +485,7 @@ func (x *TLSContext) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSContext.ProtoReflect.Descriptor instead. func (*TLSContext) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} } func (x *TLSContext) GetTrustedCa() string { @@ -433,6 +565,11 @@ type PortNetworkPolicyRule struct { // applied on the flow's remote host is contained in this set. // Optional. If not specified, any remote host is matched by this predicate. RemotePolicies []uint32 `protobuf:"varint,7,rep,packed,name=remote_policies,json=remotePolicies,proto3" json:"remote_policies,omitempty"` + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + Selectors []string `protobuf:"bytes,11,rep,name=selectors,proto3" json:"selectors,omitempty"` // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. DownstreamTlsContext *TLSContext `protobuf:"bytes,3,opt,name=downstream_tls_context,json=downstreamTlsContext,proto3" json:"downstream_tls_context,omitempty"` @@ -473,7 +610,7 @@ type PortNetworkPolicyRule struct { func (x *PortNetworkPolicyRule) Reset() { *x = PortNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +622,7 @@ func (x *PortNetworkPolicyRule) String() string { func (*PortNetworkPolicyRule) ProtoMessage() {} func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +635,7 @@ func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*PortNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} } func (x *PortNetworkPolicyRule) GetPrecedence() uint32 { @@ -554,6 +691,13 @@ func (x *PortNetworkPolicyRule) GetRemotePolicies() []uint32 { return nil } +func (x *PortNetworkPolicyRule) GetSelectors() []string { + if x != nil { + return x.Selectors + } + return nil +} + func (x *PortNetworkPolicyRule) GetDownstreamTlsContext() *TLSContext { if x != nil { return x.DownstreamTlsContext @@ -679,7 +823,7 @@ type HttpNetworkPolicyRules struct { func (x *HttpNetworkPolicyRules) Reset() { *x = HttpNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -691,7 +835,7 @@ func (x *HttpNetworkPolicyRules) String() string { func (*HttpNetworkPolicyRules) ProtoMessage() {} func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -704,7 +848,7 @@ func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} } func (x *HttpNetworkPolicyRules) GetHttpRules() []*HttpNetworkPolicyRule { @@ -729,7 +873,7 @@ type HeaderMatch struct { func (x *HeaderMatch) Reset() { *x = HeaderMatch{} - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +885,7 @@ func (x *HeaderMatch) String() string { func (*HeaderMatch) ProtoMessage() {} func (x *HeaderMatch) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -754,7 +898,7 @@ func (x *HeaderMatch) ProtoReflect() protoreflect.Message { // Deprecated: Use HeaderMatch.ProtoReflect.Descriptor instead. func (*HeaderMatch) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} } func (x *HeaderMatch) GetName() string { @@ -822,7 +966,7 @@ type HttpNetworkPolicyRule struct { func (x *HttpNetworkPolicyRule) Reset() { *x = HttpNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -834,7 +978,7 @@ func (x *HttpNetworkPolicyRule) String() string { func (*HttpNetworkPolicyRule) ProtoMessage() {} func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -847,7 +991,7 @@ func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} } func (x *HttpNetworkPolicyRule) GetHeaders() []*v31.HeaderMatcher { @@ -877,7 +1021,7 @@ type KafkaNetworkPolicyRules struct { func (x *KafkaNetworkPolicyRules) Reset() { *x = KafkaNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -889,7 +1033,7 @@ func (x *KafkaNetworkPolicyRules) String() string { func (*KafkaNetworkPolicyRules) ProtoMessage() {} func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -902,7 +1046,7 @@ func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} } func (x *KafkaNetworkPolicyRules) GetKafkaRules() []*KafkaNetworkPolicyRule { @@ -941,7 +1085,7 @@ type KafkaNetworkPolicyRule struct { func (x *KafkaNetworkPolicyRule) Reset() { *x = KafkaNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -953,7 +1097,7 @@ func (x *KafkaNetworkPolicyRule) String() string { func (*KafkaNetworkPolicyRule) ProtoMessage() {} func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -966,7 +1110,7 @@ func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} } func (x *KafkaNetworkPolicyRule) GetApiVersion() int32 { @@ -1017,7 +1161,7 @@ type L7NetworkPolicyRules struct { func (x *L7NetworkPolicyRules) Reset() { *x = L7NetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1029,7 +1173,7 @@ func (x *L7NetworkPolicyRules) String() string { func (*L7NetworkPolicyRules) ProtoMessage() {} func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1042,7 +1186,7 @@ func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRules.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} } func (x *L7NetworkPolicyRules) GetL7AllowRules() []*L7NetworkPolicyRule { @@ -1080,7 +1224,7 @@ type L7NetworkPolicyRule struct { func (x *L7NetworkPolicyRule) Reset() { *x = L7NetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1092,7 +1236,7 @@ func (x *L7NetworkPolicyRule) String() string { func (*L7NetworkPolicyRule) ProtoMessage() {} func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1105,7 +1249,7 @@ func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRule.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{12} } func (x *L7NetworkPolicyRule) GetName() string { @@ -1140,7 +1284,7 @@ type NetworkPoliciesConfigDump struct { func (x *NetworkPoliciesConfigDump) Reset() { *x = NetworkPoliciesConfigDump{} - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1152,7 +1296,7 @@ func (x *NetworkPoliciesConfigDump) String() string { func (*NetworkPoliciesConfigDump) ProtoMessage() {} func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1165,7 +1309,7 @@ func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPoliciesConfigDump.ProtoReflect.Descriptor instead. func (*NetworkPoliciesConfigDump) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{13} } func (x *NetworkPoliciesConfigDump) GetNetworkpolicies() []*NetworkPolicy { @@ -1179,7 +1323,14 @@ var File_cilium_api_npds_proto protoreflect.FileDescriptor const file_cilium_api_npds_proto_rawDesc = "" + "\n" + - "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x95\x02\n" + + "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x84\x01\n" + + "\x15NetworkPolicyResource\x12/\n" + + "\x06policy\x18\x01 \x01(\v2\x15.cilium.NetworkPolicyH\x00R\x06policy\x12.\n" + + "\bselector\x18\x02 \x01(\v2\x10.cilium.SelectorH\x00R\bselectorB\n" + + "\n" + + "\bresource\"7\n" + + "\bSelector\x12+\n" + + "\x11remote_identities\x18\x01 \x03(\rR\x10remoteIdentities\"\x95\x02\n" + "\rNetworkPolicy\x123\n" + "\fendpoint_ips\x18\x01 \x03(\tB\x10\xfaB\r\x92\x01\n" + "\b\x01\x10\x02\"\x04r\x02\x10\x01R\vendpointIps\x12\x1f\n" + @@ -1202,7 +1353,7 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\fserver_names\x18\x04 \x03(\tR\vserverNames\x12A\n" + "\x1dvalidation_context_sds_secret\x18\x05 \x01(\tR\x1avalidationContextSdsSecret\x12$\n" + "\x0etls_sds_secret\x18\x06 \x01(\tR\ftlsSdsSecret\x12%\n" + - "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\xfb\x05\n" + + "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\x99\x06\n" + "\x15PortNetworkPolicyRule\x12\x1e\n" + "\n" + "precedence\x18\n" + @@ -1212,7 +1363,8 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x04deny\x18\b \x01(\bH\x00R\x04deny\x12$\n" + "\bproxy_id\x18\t \x01(\rB\t\xfaB\x06*\x04\x18\xff\xff\x03R\aproxyId\x12\x12\n" + "\x04name\x18\x05 \x01(\tR\x04name\x12'\n" + - "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12H\n" + + "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12\x1c\n" + + "\tselectors\x18\v \x03(\tR\tselectors\x12H\n" + "\x16downstream_tls_context\x18\x03 \x01(\v2\x12.cilium.TLSContextR\x14downstreamTlsContext\x12D\n" + "\x14upstream_tls_context\x18\x04 \x01(\v2\x12.cilium.TLSContextR\x12upstreamTlsContext\x12\xa1\x01\n" + "\fserver_names\x18\x06 \x03(\tB~\xfaB{\x92\x01x\"vrt2r^(([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)[.])*([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)$R\vserverNames\x12\x19\n" + @@ -1270,7 +1422,10 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x1dNetworkPolicyDiscoveryService\x12z\n" + "\x15StreamNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x9e\x01\n" + "\x14FetchNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/v3/discovery:network_policies\x1a\x1c\x8a\xa4\x96\xf3\a\x16\n" + - "\x14cilium.NetworkPolicyB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" + "\x14cilium.NetworkPolicy2\xda\x01\n" + + "%NetworkPolicyResourceDiscoveryService\x12\x8a\x01\n" + + "\x1bDeltaNetworkPolicyResources\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a$\x8a\xa4\x96\xf3\a\x1e\n" + + "\x1ccilium.NetworkPolicyResourceB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( file_cilium_api_npds_proto_rawDescOnce sync.Once @@ -1285,59 +1440,67 @@ func file_cilium_api_npds_proto_rawDescGZIP() []byte { } var file_cilium_api_npds_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_cilium_api_npds_proto_goTypes = []any{ - (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction - (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction - (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy - (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy - (*TLSContext)(nil), // 4: cilium.TLSContext - (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule - (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules - (*HeaderMatch)(nil), // 7: cilium.HeaderMatch - (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule - (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules - (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule - (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules - (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule - (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump - nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry - (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol - (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher - (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher - (*v33.DiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DiscoveryRequest - (*v33.DiscoveryResponse)(nil), // 19: envoy.service.discovery.v3.DiscoveryResponse + (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction + (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction + (*NetworkPolicyResource)(nil), // 2: cilium.NetworkPolicyResource + (*Selector)(nil), // 3: cilium.Selector + (*NetworkPolicy)(nil), // 4: cilium.NetworkPolicy + (*PortNetworkPolicy)(nil), // 5: cilium.PortNetworkPolicy + (*TLSContext)(nil), // 6: cilium.TLSContext + (*PortNetworkPolicyRule)(nil), // 7: cilium.PortNetworkPolicyRule + (*HttpNetworkPolicyRules)(nil), // 8: cilium.HttpNetworkPolicyRules + (*HeaderMatch)(nil), // 9: cilium.HeaderMatch + (*HttpNetworkPolicyRule)(nil), // 10: cilium.HttpNetworkPolicyRule + (*KafkaNetworkPolicyRules)(nil), // 11: cilium.KafkaNetworkPolicyRules + (*KafkaNetworkPolicyRule)(nil), // 12: cilium.KafkaNetworkPolicyRule + (*L7NetworkPolicyRules)(nil), // 13: cilium.L7NetworkPolicyRules + (*L7NetworkPolicyRule)(nil), // 14: cilium.L7NetworkPolicyRule + (*NetworkPoliciesConfigDump)(nil), // 15: cilium.NetworkPoliciesConfigDump + nil, // 16: cilium.L7NetworkPolicyRule.RuleEntry + (v3.SocketAddress_Protocol)(0), // 17: envoy.config.core.v3.SocketAddress.Protocol + (*v31.HeaderMatcher)(nil), // 18: envoy.config.route.v3.HeaderMatcher + (*v32.MetadataMatcher)(nil), // 19: envoy.type.matcher.v3.MetadataMatcher + (*v33.DiscoveryRequest)(nil), // 20: envoy.service.discovery.v3.DiscoveryRequest + (*v33.DeltaDiscoveryRequest)(nil), // 21: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v33.DiscoveryResponse)(nil), // 22: envoy.service.discovery.v3.DiscoveryResponse + (*v33.DeltaDiscoveryResponse)(nil), // 23: envoy.service.discovery.v3.DeltaDiscoveryResponse } var file_cilium_api_npds_proto_depIdxs = []int32{ - 3, // 0: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 3, // 1: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 15, // 2: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol - 5, // 3: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule - 4, // 4: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext - 4, // 5: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext - 6, // 6: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules - 9, // 7: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules - 11, // 8: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules - 8, // 9: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule - 0, // 10: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction - 1, // 11: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction - 16, // 12: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher - 7, // 13: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch - 10, // 14: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule - 12, // 15: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule - 12, // 16: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule - 14, // 17: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry - 17, // 18: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher - 2, // 19: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy - 18, // 20: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 18, // 21: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 19, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 19, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 22, // [22:24] is the sub-list for method output_type - 20, // [20:22] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 4, // 0: cilium.NetworkPolicyResource.policy:type_name -> cilium.NetworkPolicy + 3, // 1: cilium.NetworkPolicyResource.selector:type_name -> cilium.Selector + 5, // 2: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 5, // 3: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 17, // 4: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol + 7, // 5: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule + 6, // 6: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext + 6, // 7: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext + 8, // 8: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules + 11, // 9: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules + 13, // 10: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules + 10, // 11: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule + 0, // 12: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction + 1, // 13: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction + 18, // 14: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher + 9, // 15: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch + 12, // 16: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule + 14, // 17: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule + 14, // 18: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule + 16, // 19: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry + 19, // 20: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher + 4, // 21: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy + 20, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 20, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 24: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 22, // 25: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 22, // 26: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 27: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 25, // [25:28] is the sub-list for method output_type + 22, // [22:25] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_cilium_api_npds_proto_init() } @@ -1345,7 +1508,11 @@ func file_cilium_api_npds_proto_init() { if File_cilium_api_npds_proto != nil { return } - file_cilium_api_npds_proto_msgTypes[3].OneofWrappers = []any{ + file_cilium_api_npds_proto_msgTypes[0].OneofWrappers = []any{ + (*NetworkPolicyResource_Policy)(nil), + (*NetworkPolicyResource_Selector)(nil), + } + file_cilium_api_npds_proto_msgTypes[5].OneofWrappers = []any{ (*PortNetworkPolicyRule_PassPrecedence)(nil), (*PortNetworkPolicyRule_Deny)(nil), (*PortNetworkPolicyRule_HttpRules)(nil), @@ -1358,9 +1525,9 @@ func file_cilium_api_npds_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_npds_proto_rawDesc), len(file_cilium_api_npds_proto_rawDesc)), NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, - NumServices: 1, + NumServices: 2, }, GoTypes: file_cilium_api_npds_proto_goTypes, DependencyIndexes: file_cilium_api_npds_proto_depIdxs, diff --git a/go/cilium/api/npds.pb.validate.go b/go/cilium/api/npds.pb.validate.go index 418649f39..ea344bc3f 100644 --- a/go/cilium/api/npds.pb.validate.go +++ b/go/cilium/api/npds.pb.validate.go @@ -39,6 +39,294 @@ var ( _ = corev3.SocketAddress_Protocol(0) ) +// Validate checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *NetworkPolicyResource) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// NetworkPolicyResourceMultiError, or nil if none found. +func (m *NetworkPolicyResource) ValidateAll() error { + return m.validate(true) +} + +func (m *NetworkPolicyResource) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + switch v := m.Resource.(type) { + case *NetworkPolicyResource_Policy: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetPolicy()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetPolicy()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + } + } + } + + case *NetworkPolicyResource_Selector: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetSelector()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetSelector()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + } + } + } + + default: + _ = v // ensures v is used + } + + if len(errors) > 0 { + return NetworkPolicyResourceMultiError(errors) + } + + return nil +} + +// NetworkPolicyResourceMultiError is an error wrapping multiple validation +// errors returned by NetworkPolicyResource.ValidateAll() if the designated +// constraints aren't met. +type NetworkPolicyResourceMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m NetworkPolicyResourceMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m NetworkPolicyResourceMultiError) AllErrors() []error { return m } + +// NetworkPolicyResourceValidationError is the validation error returned by +// NetworkPolicyResource.Validate if the designated constraints aren't met. +type NetworkPolicyResourceValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e NetworkPolicyResourceValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e NetworkPolicyResourceValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e NetworkPolicyResourceValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e NetworkPolicyResourceValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e NetworkPolicyResourceValidationError) ErrorName() string { + return "NetworkPolicyResourceValidationError" +} + +// Error satisfies the builtin error interface +func (e NetworkPolicyResourceValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sNetworkPolicyResource.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = NetworkPolicyResourceValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = NetworkPolicyResourceValidationError{} + +// Validate checks the field values on Selector with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Selector) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Selector with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SelectorMultiError, or nil +// if none found. +func (m *Selector) ValidateAll() error { + return m.validate(true) +} + +func (m *Selector) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SelectorMultiError(errors) + } + + return nil +} + +// SelectorMultiError is an error wrapping multiple validation errors returned +// by Selector.ValidateAll() if the designated constraints aren't met. +type SelectorMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SelectorMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SelectorMultiError) AllErrors() []error { return m } + +// SelectorValidationError is the validation error returned by +// Selector.Validate if the designated constraints aren't met. +type SelectorValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SelectorValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SelectorValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SelectorValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SelectorValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SelectorValidationError) ErrorName() string { return "SelectorValidationError" } + +// Error satisfies the builtin error interface +func (e SelectorValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSelector.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SelectorValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SelectorValidationError{} + // Validate checks the field values on NetworkPolicy with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. diff --git a/go/cilium/api/npds_grpc.pb.go b/go/cilium/api/npds_grpc.pb.go index 600fdfa95..cd9b2c32e 100644 --- a/go/cilium/api/npds_grpc.pb.go +++ b/go/cilium/api/npds_grpc.pb.go @@ -29,6 +29,7 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceClient interface { StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicies(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) @@ -70,6 +71,7 @@ func (c *networkPolicyDiscoveryServiceClient) FetchNetworkPolicies(ctx context.C // for forward compatibility. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceServer interface { StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicies(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) @@ -158,3 +160,104 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, Metadata: "cilium/api/npds.proto", } + +const ( + NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/DeltaNetworkPolicyResources" +) + +// NetworkPolicyResourceDiscoveryServiceClient is the client API for NetworkPolicyResourceDiscoveryService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Policy and selector resource names are exact-match identifiers in delta NPDS. +type NetworkPolicyResourceDiscoveryServiceClient interface { + DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) +} + +type networkPolicyResourceDiscoveryServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNetworkPolicyResourceDiscoveryServiceClient(cc grpc.ClientConnInterface) NetworkPolicyResourceDiscoveryServiceClient { + return &networkPolicyResourceDiscoveryServiceClient{cc} +} + +func (c *networkPolicyResourceDiscoveryServiceClient) DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryServiceServer is the server API for NetworkPolicyResourceDiscoveryService service. +// All implementations must embed UnimplementedNetworkPolicyResourceDiscoveryServiceServer +// for forward compatibility. +// +// Policy and selector resource names are exact-match identifiers in delta NPDS. +type NetworkPolicyResourceDiscoveryServiceServer interface { + DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +// UnimplementedNetworkPolicyResourceDiscoveryServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedNetworkPolicyResourceDiscoveryServiceServer struct{} + +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() { +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) testEmbeddedByValue() {} + +// UnsafeNetworkPolicyResourceDiscoveryServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NetworkPolicyResourceDiscoveryServiceServer will +// result in compilation errors. +type UnsafeNetworkPolicyResourceDiscoveryServiceServer interface { + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +func RegisterNetworkPolicyResourceDiscoveryServiceServer(s grpc.ServiceRegistrar, srv NetworkPolicyResourceDiscoveryServiceServer) { + // If the following call panics, it indicates UnimplementedNetworkPolicyResourceDiscoveryServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&NetworkPolicyResourceDiscoveryService_ServiceDesc, srv) +} + +func _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).DeltaNetworkPolicyResources(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyResourceDiscoveryService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NetworkPolicyResourceDiscoveryService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "cilium.NetworkPolicyResourceDiscoveryService", + HandlerType: (*NetworkPolicyResourceDiscoveryServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "DeltaNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "cilium/api/npds.proto", +} diff --git a/tests/BUILD b/tests/BUILD index 594300e99..5c28af376 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -121,6 +121,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "versioned_test", + srcs = ["versioned_test.cc"], + repository = "@envoy", + deps = [ + "//cilium:versioned_lib", + ], +) + envoy_cc_test( name = "metadata_config_test", srcs = ["metadata_config_test.cc"], diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index 222fad4e9..f638b47cf 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -72,7 +72,7 @@ class CiliumNetworkPolicyTest : public ::testing::Test { ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, TlsSessionTicketKeysContext, TlsSessionTicketKeys); ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, GenericSecret, GenericSecret); - policy_map_ = std::make_shared(factory_context_); + policy_map_ = std::make_shared(factory_context_, false, useDeltaXds()); } void TearDown() override { @@ -80,6 +80,8 @@ class CiliumNetworkPolicyTest : public ::testing::Test { policy_map_.reset(); } + virtual bool useDeltaXds() const { return false; } + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacks() const { return policy_map_->subscriptionCallbacksForTest(); } @@ -99,6 +101,24 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return message.version_info(); } + std::string deltaUpdateFromYaml(const std::string& config) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); + NetworkPolicyResourceDecoder network_policy_resource_decoder( + ProtobufMessage::getNullValidationVisitor(), "name"); + auto decoded_resources = std::make_unique(); + for (const auto& resource : message.resources()) { + decoded_resources->pushBack( + Config::DecodedResourceImpl::fromResource(network_policy_resource_decoder, resource)); + } + + EXPECT_TRUE(subscriptionCallbacks() + .onConfigUpdate(decoded_resources->refvec_, message.removed_resources(), + message.system_version_info()) + .ok()); + return message.system_version_info(); + } + testing::AssertionResult validate(const std::string& pod_ip, const std::string& expected) { const auto& policy = policy_map_->getPolicyInstance(pod_ip, false); auto str = policy.string(); @@ -218,6 +238,20 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return policy_map_->statsForTest().updates_rejected_.name(); } + PolicyInstanceConstSharedPtr policyInstanceShared(const std::string& pod_ip) const { + return policy_map_->getPolicyInstanceSharedForTest(pod_ip); + } + + uint64_t selectorStreamGenerationForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorStreamGenerationForTest(policy); + } + + SelectorVersion selectorVersionForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorVersionForTest(policy); + } + + void resetStreamForTest() { policy_map_->resetStreamForTest(); } + NiceMock factory_context_; NiceMock secret_manager_; std::shared_ptr policy_map_; @@ -225,6 +259,11 @@ class CiliumNetworkPolicyTest : public ::testing::Test { uint16_t proxy_id_ = 42; }; +class CiliumNetworkPolicyDeltaTest : public CiliumNetworkPolicyTest { +protected: + bool useDeltaXds() const override { return true; } +}; + TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { EXPECT_EQ("cilium.policy.updates_rejected", updatesRejectedStatName()); } @@ -242,6 +281,1379 @@ TEST_F(CiliumNetworkPolicyTest, SimplePolicyUpdate) { EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } +TEST_F(CiliumNetworkPolicyTest, RejectsWhitespaceInSotwWrappedResourceName) { + EXPECT_THROW_WITH_MESSAGE(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/envoy.service.discovery.v3.Resource + name: "policy 42" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF"), + EnvoyException, + "Network Policy resource name 'policy 42' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaIncrementalPolicyUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 8080)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSameStreamKeepsUntouchedResources) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPolicyByResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + - "f00d::1" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("f00d::1")); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("f00d::1", 43, 80)); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "policy-42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_FALSE(policy_map_->exists("f00d::1")); + EXPECT_FALSE(validate("10.1.2.3", "")); + EXPECT_FALSE(validate("f00d::1", "")); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateTakesEffectImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPolicyUpdateRejectsMissingSelectorResource) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-2" ] +)EOF"), + EnvoyException, + "Delta Network Policy rule references missing selector resource " + "'selector-2'"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameDoesNotRebindOldPolicy) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameInSameUpdateActsAsUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBehavior) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.3'"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPassUsesCurrentSelectorMembership) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaWildcardPortPassIsMergedToExactPortRules) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 0 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - port: 80 + rules: + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSamePrecedenceDenyWinsOverPass) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 1000 + deny: true + selectors: [ "selector-1" ] + - precedence: 500 + selectors: [ "selector-1" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateChangesPassBehaviorImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyTest, SotwRejectsSelectorsInRules) { + EXPECT_THROW(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitrarySelectorResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "7" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "7" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEmbeddedRemotePoliciesInRules) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 43 ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsAddedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector 1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "Network Policy delta resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +removed_resources: +- "selector 1" +)EOF"), + EnvoyException, + "Network Policy delta removed resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key 'shared-name'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.3'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.4'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46, 47 ] +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key " + "'shared-selector'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-qualified-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPoliciesByArbitraryResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-qualified-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistingPolicies) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.3'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsPolicyResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.3'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsEndpointIpCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.4'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key 'shared-name'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "Network Policy delta update has duplicate resource key '10.1.2.3'"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.3" +)EOF"), + EnvoyException, + "Network Policy delta removed resource '10.1.2.3' is a policy endpoint IP alias, not a " + "resource name"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAllowsEndpointIpReusingRemovedPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.4" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesWithNumericSuffixes) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-042" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsZeroEndpointIdRegardlessOfResourceName) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 0 +)EOF"), + EnvoyException, "Network Policy endpoint_id must be non-zero"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsInconsistentPassPrecedence) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 100 + selectors: [ "selector-1" ] + - precedence: 900 + pass_precedence: 200 + selectors: [ "selector-2" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamReplacesStateWithFullSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + + EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + + EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +)EOF")); + + EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "selector-2" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto new_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, new_policy); + EXPECT_NE(old_policy.get(), new_policy.get()); + + EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_EQ(2, selectorStreamGenerationForTest(*new_policy)); + EXPECT_EQ(3, selectorVersionForTest(*new_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + TEST_F(CiliumNetworkPolicyTest, OverlappingPortRange) { EXPECT_NO_THROW(updateFromYaml(R"EOF(version_info: "1" resources: diff --git a/tests/versioned_test.cc b/tests/versioned_test.cc new file mode 100644 index 000000000..be90c9ec5 --- /dev/null +++ b/tests/versioned_test.cc @@ -0,0 +1,1165 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "source/common/common/lock_guard.h" +#include "source/common/common/thread.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "cilium/versioned.h" + +// NOLINT(namespace-envoy) +namespace { + +class TrackedValue : public VersionedNode { +public: + explicit TrackedValue(int id) : TrackedValue(id, id) {} + + TrackedValue(int key_id, int generation) + : key_id_(key_id), generation_(generation), unique_id_(next_unique_id_++) { + ++constructed_count_; + ++live_count_; + } + + ~TrackedValue() { + ++destroyed_count_; + --live_count_; + ++destroyed_by_id_[generation_]; + } + + int id() const { return generation_; } + int keyId() const { return key_id_; } + int generation() const { return generation_; } + uint64_t uniqueId() const { return unique_id_; } + + static void resetCounters() { + constructed_count_ = 0; + destroyed_count_ = 0; + live_count_ = 0; + destroyed_by_id_.clear(); + next_unique_id_ = 1; + } + + static int constructedCount() { return constructed_count_; } + static int destroyedCount() { return destroyed_count_; } + static int liveCount() { return live_count_; } + + static int destroyedCountFor(int id) { + const auto it = destroyed_by_id_.find(id); + return it == destroyed_by_id_.end() ? 0 : it->second; + } + +private: + int key_id_; + int generation_; + uint64_t unique_id_; + + static inline int constructed_count_ = 0; + static inline int destroyed_count_ = 0; + static inline int live_count_ = 0; + static inline uint64_t next_unique_id_ = 1; + static inline std::map destroyed_by_id_; +}; + +using TrackedHandle = VersionedHandle; +using TrackedDeferredDeletion = DeferredDeletion; +using TrackedMap = VersionedMap; +using TrackedAccess = VersionedTestAccess; + +const TrackedValue* activeValue(const TrackedMap& map, const std::string& key, + uint64_t version = versionMax) { + auto handle = map.find(key); + return handle != nullptr ? handle->get(version) : nullptr; +} + +struct PublishedHandles { + uint64_t snapshot_id; + uint64_t version; + std::vector> handles; +}; + +struct ObservedNode { + const void* node; + const void* next; + const TrackedValue* value; + uint64_t add_version; + uint64_t remove_version; +}; + +enum class ValidationMode { + StrictConsistentView, + ConcurrentSafeInvisibleTail, +}; + +std::string describeValue(const TrackedValue* value) { + if (value == nullptr) { + return "nullptr"; + } + return testing::PrintToString( + std::make_tuple(value->keyId(), value->generation(), value->uniqueId())); +} + +std::string describeObservedChain(const std::vector& nodes) { + std::ostringstream out; + for (size_t i = 0; i < nodes.size(); ++i) { + if (i > 0) { + out << " -> "; + } + const auto& node = nodes[i]; + out << "{node=" << node.node << ", next=" << node.next + << ", value=" << describeValue(node.value) << ", add=" << node.add_version + << ", remove=" << node.remove_version << "}"; + } + if (nodes.empty()) { + out << ""; + } + return out.str(); +} + +std::string validateTraversalAtVersion(const std::string& label, + const VersionedNode* initial_head, + uint64_t version, ValidationMode mode, + const TrackedValue* expected = nullptr, + const void* handle = nullptr) { + const bool require_consistent_view = mode == ValidationMode::StrictConsistentView; + const TrackedValue* visible_value = nullptr; + size_t visible_count = 0; + uint64_t previous_add_version = versionNotRemoved; + bool in_invisible_tail = false; + absl::flat_hash_map visited_nodes; + std::vector observed_nodes; + for (const auto* node = initial_head; node != nullptr;) { + const auto* next = TrackedAccess::next(node); + observed_nodes.push_back(ObservedNode{node, next, TrackedAccess::value(node), + TrackedAccess::addVersion(node), + TrackedAccess::removeVersion(node)}); + const auto add_version = observed_nodes.back().add_version; + const auto [visited_it, inserted] = visited_nodes.emplace(node, add_version); + if (!inserted) { + if (visited_it->second == add_version) { + return "cycle detected for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + visited_it->second = add_version; + } + + const auto remove_version = observed_nodes.back().remove_version; + if (add_version >= versionNotRemoved) { + return "unpublished node observed for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if ((require_consistent_view || !in_invisible_tail) && add_version > previous_add_version) { + return "add_version increased while following next links for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " previous_add=" + std::to_string(previous_add_version) + + " current_add=" + std::to_string(add_version) + + " chain=" + describeObservedChain(observed_nodes); + } + if (remove_version < versionNotRemoved && add_version > remove_version) { + return "invalid node visibility window for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + + if (add_version <= version && version < remove_version) { + ++visible_count; + visible_value = TrackedAccess::value(node); + if (require_consistent_view && visible_count > 1) { + return "multiple visible nodes at version " + std::to_string(version) + " for '" + label + + "' handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + } + // Once traversal reaches the first node that is already stale for this version, production + // readers no longer depend on per-handle ordering. First-phase GC may have already rewired the + // rest of the tail into a mixed deferred-deletion list, but those nodes must remain safe to + // walk and invisible for this version. + if (!require_consistent_view && in_invisible_tail && + TrackedAccess::isVisibleInVersion(node, version)) { + return "visible node observed after entering invisible tail for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if (!TrackedAccess::isVisibleInVersion(node, version) && + !TrackedAccess::isAddedAfterVersion(node, version)) { + in_invisible_tail = true; + } + previous_add_version = add_version; + node = next; + } + + if (require_consistent_view && expected != visible_value) { + return "handle->get mismatch for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " expected=" + describeValue(expected) + ", traversed=" + describeValue(visible_value) + + ", chain=" + describeObservedChain(observed_nodes); + } + + return ""; +} + +std::string validateHandleAtVersion(const std::string& label, const TrackedHandle& handle, + uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + if (handle == nullptr) { + return "snapshot contains null handle for '" + label + "'"; + } + + return validateTraversalAtVersion( + label, TrackedAccess::head(handle), version, mode, + mode == ValidationMode::StrictConsistentView ? handle->get(version) : nullptr, handle.get()); +} + +std::string +validateNodeAtVersion(const std::string& label, const VersionedNode* node, + uint64_t version, + ValidationMode mode = ValidationMode::ConcurrentSafeInvisibleTail) { + if (node == nullptr) { + return "null node supplied for '" + label + "'"; + } + return validateTraversalAtVersion(label, node, version, mode); +} + +std::string +validatePublishedSnapshotAtVersion(const PublishedHandles& snapshot, uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + for (const auto& [label, handle] : snapshot.handles) { + auto error = validateHandleAtVersion(label, handle, version, mode); + if (!error.empty()) { + return error; + } + } + return ""; +} + +std::string validatePublishedSnapshot(const PublishedHandles& snapshot) { + return validatePublishedSnapshotAtVersion(snapshot, snapshot.version); +} + +std::shared_ptr makePublishedHandles(const TrackedMap& map, + uint64_t snapshot_id = 0) { + auto snapshot = std::make_shared(); + snapshot->snapshot_id = snapshot_id; + snapshot->version = map.getVersion(); + + absl::flat_hash_set seen_handles; + for (const auto& [key, handle] : TrackedAccess::entries(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back(key, handle); + } + } + size_t dirty_index = 0; + for (const auto& handle : TrackedAccess::dirtyValues(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back("dirty:" + std::to_string(dirty_index++), handle); + } + } + + return snapshot; +} + +class VersionedTest : public testing::Test { +protected: + void SetUp() override { TrackedValue::resetCounters(); } + + void TearDown() override { + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::constructedCount(), TrackedValue::destroyedCount()); + } +}; + +TEST_F(VersionedTest, VersionedValueFirstInsertedValueIsVisibleInPublishedVersion) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + + EXPECT_EQ(value.get(0), nullptr); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); +} + +TEST_F(VersionedTest, VersionedValueNewerValueShadowsOlderValueFromNewVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + EXPECT_EQ(TrackedValue::liveCount(), 2); +} + +TEST_F(VersionedTest, VersionedValueClearHidesOnlyFromThatVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + EXPECT_EQ(value.get(2), nullptr); +} + +TEST_F(VersionedTest, VersionedValueRevertRestoresPendingClear) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + EXPECT_EQ(value.get(2), nullptr); + + value.revert(1, deferred); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 1); +} + +TEST_F(VersionedTest, VersionedValueRevertDeletesUnpublishedReplacement) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); +} + +TEST_F(VersionedTest, VersionedValueDeferredRevertDeletesReplacementOnBatchDestruction) { + { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedValueGcDefersDeletionUntilBatchDestruction) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion deferred; + EXPECT_FALSE(value.gcForVersion(1, deferred)); + + EXPECT_TRUE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + { + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); +} + +TEST_F(VersionedTest, DeferredDeletionMoveTransfersDeletionOwnership) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion moved = std::move(deferred); + EXPECT_FALSE(moved.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, FirstPhaseGcKeepsCurrentAndHistoricalLookupsMemorySafe) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(validateHandleAtVersion("a", handle, version1), ""); + EXPECT_EQ(validateHandleAtVersion("a", handle, version2), ""); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, ConcurrentValidatorAcceptsMixedDeferredTailTraversal) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 1)); + map.insert("b", new TrackedValue(2, 1)); + map.publishNextVersion(); + + auto handle_a = map.find("a"); + auto handle_b = map.find("b"); + ASSERT_NE(handle_a, nullptr); + ASSERT_NE(handle_b, nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 2)); + map.insert("b", new TrackedValue(2, 2)); + const auto version2 = map.publishNextVersion(); + + const auto* stale_a = TrackedAccess::next(TrackedAccess::head(handle_a)); + const auto* stale_b = TrackedAccess::next(TrackedAccess::head(handle_b)); + ASSERT_NE(stale_a, nullptr); + ASSERT_NE(stale_b, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + + EXPECT_EQ(validateHandleAtVersion("a", handle_a, version2), ""); + EXPECT_EQ(validateHandleAtVersion("b", handle_b, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-a", stale_a, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-b", stale_b, version2), ""); + } +} + +TEST_F(VersionedTest, VersionedMapGcRetainsDirtyHandleUntilAllFutureRemovalsAreReclaimed) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(3)); + const auto version3 = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 3); + + { + auto deferred = map.gc(version2); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 3); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + + { + auto deferred = map.gc(version3); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); +} + +TEST_F(VersionedTest, VersionedMapVersionStartsAtMin) { + TrackedMap map; + + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapPublishWithNoPendingChangesReturnsZero) { + TrackedMap map; + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + + map.prepareNextVersion(); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapInsertNewKeyThenRevertRemovesItEntirely) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + + ASSERT_NE(activeValue(map, "a", 1), nullptr); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(map.find("a"), nullptr); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesNewKeyAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, VersionedMapUpdateExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + EXPECT_EQ(map.getVersion(), version1); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesReplacementAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedMapClearExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.clear("a"); + + EXPECT_EQ(activeValue(map, "a"), nullptr); + + auto deferred = map.revert(); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapClearMissingKeyIsNoOp) { + TrackedMap map; + + map.prepareNextVersion(); + map.clear("missing"); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + EXPECT_EQ(TrackedValue::liveCount(), 0); +} + +TEST_F(VersionedTest, VersionedMapSnapshotsIsolateKeysAndVersions) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.insert("b", new TrackedValue(10)); + auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + const auto version2 = map.getVersion(); + + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "b", version1), nullptr); + EXPECT_EQ(activeValue(map, "b", version1)->id(), 10); + ASSERT_NE(activeValue(map, "b", version2), nullptr); + EXPECT_EQ(activeValue(map, "b", version2)->id(), 10); + + map.gc(published_version); +} + +TEST_F(VersionedTest, VersionedMapActiveUpdateReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapClearAndReinsertSameKeyBeforePublishReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.clear("a"); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapRemovedHandleStaysEmptyAfterSameNameReAdd) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + auto old_handle = map.find("a"); + ASSERT_NE(old_handle, nullptr); + + map.prepareNextVersion(); + map.clear("a"); + const auto version2 = map.publishNextVersion(); + map.gc(version2); + + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(old_handle->get(version2), nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version3 = map.publishNextVersion(); + + auto new_handle = map.find("a"); + ASSERT_NE(new_handle, nullptr); + EXPECT_NE(new_handle, old_handle); + ASSERT_NE(new_handle->get(version3), nullptr); + EXPECT_EQ(new_handle->get(version3)->id(), 2); + EXPECT_EQ(old_handle->get(version3), nullptr); + map.gc(version3); +} + +TEST_F(VersionedTest, VersionedMapValidatorsAcceptStablePublishedSnapshots) { + TrackedMap map; + std::array generations{}; + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + auto expect_snapshot_valid = [&](const std::string& phase) { + auto snapshot = makePublishedHandles(map); + EXPECT_EQ(validatePublishedSnapshot(*snapshot), "") << phase; + }; + + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); + map.insert("key-11", next_value(11)); + const auto version1 = map.publishNextVersion(); + ASSERT_EQ(version1, 1); + expect_snapshot_valid("after initial publish"); + + // Exercise same-version churn on one handle: update, clear, and re-add in one transaction. + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); // generation 2 + map.clear("key-10"); + map.insert("key-10", next_value(10)); // generation 3 + const auto version2 = map.publishNextVersion(); + ASSERT_EQ(version2, 2); + expect_snapshot_valid("after same-version clear and re-add publish"); + + // Exercise both the post-unlink/pre-deletion state and the stable post-deletion state. + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + expect_snapshot_valid("after first-phase gc of same-version churn publish"); + } + expect_snapshot_valid("after deferred deletion of same-version churn publish"); + + map.prepareNextVersion(); + map.clear("key-11"); + map.insert("key-12", next_value(12)); + const auto version3 = map.publishNextVersion(); + ASSERT_EQ(version3, 3); + expect_snapshot_valid("after mixed clear and insert publish"); + + { + auto deferred = map.gc(version3); + expect_snapshot_valid("after first-phase final gc"); + } + expect_snapshot_valid("after final deferred deletion"); +} + +TEST_F(VersionedTest, VersionedMapChaosConcurrentReadersAndWriter) { + TrackedMap map; + + constexpr size_t k_reader_count = 4; + constexpr int k_key_count = 32; + constexpr int k_initial_key_count = 16; + constexpr int k_max_ops_per_transaction = 8; + constexpr int k_transaction_count = 2000; + constexpr uint32_t k_seed = 1337; + + std::array key_names; + std::array generations{}; + for (int i = 0; i < k_key_count; ++i) { + key_names[i] = "key-" + std::to_string(i); + } + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + + map.prepareNextVersion(); + for (int i = 0; i < k_initial_key_count; ++i) { + map.insert(key_names[i], next_value(i)); + } + auto published_version = map.publishNextVersion(); + ASSERT_GT(published_version, 0); + + uint64_t next_snapshot_id = 1; + std::shared_ptr published_snapshot_owner = + makePublishedHandles(map, next_snapshot_id++); + std::shared_ptr previous_snapshot_owner; + std::atomic published_snapshot{published_snapshot_owner.get()}; + std::array, k_reader_count> reader_quiesced_versions; + std::array, k_reader_count> reader_snapshot_ids; + for (auto& reader_version : reader_quiesced_versions) { + reader_version.store(versionMin, std::memory_order_relaxed); + } + for (auto& reader_snapshot_id : reader_snapshot_ids) { + reader_snapshot_id.store(0, std::memory_order_relaxed); + } + + std::atomic stop{false}; + std::atomic failed{false}; + Envoy::Thread::MutexBasicLockable failure_mutex; + std::string failure_message; + Envoy::Thread::MutexBasicLockable history_mutex; + std::vector recent_history; + + auto record_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + Envoy::Thread::LockGuard lock(failure_mutex); + failure_message = message; + } + }; + + auto record_history = [&](std::string message) { + Envoy::Thread::LockGuard lock(history_mutex); + recent_history.push_back(std::move(message)); + constexpr size_t k_history_limit = 128; + if (recent_history.size() > k_history_limit) { + recent_history.erase(recent_history.begin(), + recent_history.begin() + (recent_history.size() - k_history_limit)); + } + }; + + std::vector readers; + readers.reserve(k_reader_count); + for (size_t i = 0; i < k_reader_count; ++i) { + readers.emplace_back([&, i]() { + while (!stop.load(std::memory_order_acquire)) { + const auto* snapshot = published_snapshot.load(std::memory_order_acquire); + if (snapshot == nullptr) { + continue; + } + if (const auto error = validatePublishedSnapshotAtVersion( + *snapshot, snapshot->version, ValidationMode::ConcurrentSafeInvisibleTail); + !error.empty()) { + record_failure("reader " + std::to_string(i) + + " snapshot_id=" + std::to_string(snapshot->snapshot_id) + ": " + error); + break; + } + reader_quiesced_versions[i].store(snapshot->version, std::memory_order_release); + reader_snapshot_ids[i].store(snapshot->snapshot_id, std::memory_order_release); + } + }); + } + + auto publish_snapshot = [&]() { + previous_snapshot_owner = std::move(published_snapshot_owner); + published_snapshot_owner = makePublishedHandles(map, next_snapshot_id++); + published_snapshot.store(published_snapshot_owner.get(), std::memory_order_release); + return published_snapshot_owner; + }; + + auto release_previous_snapshot = [&]() { previous_snapshot_owner.reset(); }; + + auto wait_for_readers_to_observe_snapshot = + [&](const std::shared_ptr& snapshot) { + while (!failed.load(std::memory_order_acquire)) { + bool all_quiesced = true; + for (size_t i = 0; i < k_reader_count; ++i) { + if (reader_quiesced_versions[i].load(std::memory_order_acquire) < snapshot->version || + reader_snapshot_ids[i].load(std::memory_order_acquire) < snapshot->snapshot_id) { + all_quiesced = false; + break; + } + } + if (all_quiesced) { + return true; + } + std::this_thread::yield(); + } + return false; + }; + + auto run_deferred_gc = [&](uint64_t version) { + record_history("gc phase1 " + std::to_string(version)); + auto deferred = map.gc(version); + if (deferred.empty()) { + return true; + } + + const auto post_gc_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(post_gc_snapshot)) { + return false; + } + release_previous_snapshot(); + record_history("gc phase2 " + std::to_string(version)); + return true; + }; + + auto build_active_set = [&]() { + std::array active{}; + for (int i = 0; i < k_key_count; ++i) { + active[i] = map.find(key_names[i]) != nullptr; + } + return active; + }; + + auto random_matching_key = [&](std::minstd_rand& rng, const std::array& active, + bool want_active) { + std::vector candidates; + candidates.reserve(k_key_count); + for (int i = 0; i < k_key_count; ++i) { + if (active[i] == want_active) { + candidates.push_back(i); + } + } + if (candidates.empty()) { + return -1; + } + return candidates[rng() % candidates.size()]; + }; + + std::minstd_rand rng(k_seed); + uint64_t transaction_index = 0; + while (transaction_index < k_transaction_count && !failed.load(std::memory_order_acquire)) { + auto active = build_active_set(); + const auto next_version = map.prepareNextVersion(); + ++transaction_index; + record_history("tx " + std::to_string(transaction_index) + + " prepare next=" + std::to_string(next_version)); + + const int op_count = 1 + (rng() % k_max_ops_per_transaction); + for (int op_index = 0; op_index < op_count; ++op_index) { + const bool has_active = std::ranges::any_of(active, [](bool is_active) { return is_active; }); + const bool has_absent = + std::ranges::any_of(active, [](bool is_active) { return !is_active; }); + + std::vector available_ops; + if (has_active) { + available_ops.push_back(0); // update existing + available_ops.push_back(1); // clear existing + available_ops.push_back(3); // clear and re-add + } + if (has_absent) { + available_ops.push_back(2); // insert absent + } + ASSERT_FALSE(available_ops.empty()); + + switch (available_ops[rng() % available_ops.size()]) { + case 0: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": update " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + break; + } + case 1: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear " + key_names[key_id]); + map.clear(key_names[key_id]); + active[key_id] = false; + break; + } + case 2: { + const int key_id = random_matching_key(rng, active, false); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": insert " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + case 3: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear+insert " + key_names[key_id] + + " -> gen " + std::to_string(generations[key_id] + 1)); + map.clear(key_names[key_id]); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + default: + FAIL() << "unexpected operation"; + } + } + + if ((rng() % 5) == 0) { + record_history("tx " + std::to_string(transaction_index) + " revert"); + auto deferred = map.revert(); + if (!deferred.empty()) { + const auto reverted_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(reverted_snapshot)) { + break; + } + record_history("revert deferred delete"); + } + continue; + } + + published_version = map.publishNextVersion(); + record_history("tx " + std::to_string(transaction_index) + " publish -> " + + std::to_string(published_version)); + if (published_version == 0) { + continue; + } + + const auto next_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(next_snapshot)) { + break; + } + release_previous_snapshot(); + if (!run_deferred_gc(published_version)) { + break; + } + } + + if (!failed.load(std::memory_order_acquire)) { + const auto final_version = map.getVersion(); + const auto final_snapshot = publish_snapshot(); + if (final_version > versionMin) { + EXPECT_TRUE(wait_for_readers_to_observe_snapshot(final_snapshot)); + release_previous_snapshot(); + if (!failed.load(std::memory_order_acquire)) { + EXPECT_TRUE(run_deferred_gc(final_version)); + } + } + } + + stop.store(true, std::memory_order_release); + for (auto& reader : readers) { + reader.join(); + } + published_snapshot.store(nullptr, std::memory_order_release); + previous_snapshot_owner.reset(); + published_snapshot_owner.reset(); + + if (failed.load(std::memory_order_acquire)) { + Envoy::Thread::LockGuard lock(failure_mutex); + std::ostringstream history; + { + Envoy::Thread::LockGuard history_lock(history_mutex); + for (const auto& entry : recent_history) { + history << "\n " << entry; + } + } + FAIL() << "seed=" << k_seed << " " << failure_message << "\nrecent history:" << history.str(); + } +} + +TEST_F(VersionedTest, VersionedMapGcAndDestructorDeleteValuesExactlyOnce) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + auto deferred = map.gc(published_version); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +} // namespace From 2cbdab2d024dcf3ea53b8775e95b8538e468be2b Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 16 Apr 2026 00:11:36 +0200 Subject: [PATCH 08/14] policy: Allow use_delta_npds override Store the latest desired NPDS mode in the policy map and use it for: - initial policy map subscription - re-subscription when connection under current subscription is terminated - a healthy network policy stream is not disrupted This should work for Cilium Agent upgrades and downgrades, as the agent expresses the desired mode, and listens for both. Signed-off-by: Jarno Rajahalme --- cilium/bpf_metadata.cc | 5 +- cilium/grpc_subscription.cc | 78 ++++++++++-- cilium/grpc_subscription.h | 8 +- cilium/network_policy.cc | 176 +++++++++++++++++++++++----- cilium/network_policy.h | 11 ++ tests/cilium_network_policy_test.cc | 153 ++++++++++++++++++++++++ 6 files changed, 388 insertions(+), 43 deletions(-) diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index 084be3f4f..3bae20cd9 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -264,10 +264,7 @@ Config::Config(const ::cilium::BpfMetadata& config, return std::make_shared(context, true, config.use_delta_npds()); }); - if (npmap_->useDeltaXds() != config.use_delta_npds()) { - throw EnvoyException( - "cilium.bpf_metadata: use_npds_delta must be consistent across listeners"); - } + npmap_->setUseDeltaXds(config.use_delta_npds()); } } diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index d9109abad..3e8ea4956 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -48,42 +49,85 @@ class StreamTrackedGrpcMux { public: virtual ~StreamTrackedGrpcMux() = default; virtual uint64_t streamGeneration() const = 0; + virtual bool streamConnected() const = 0; }; class SotwGrpcMuxImpl : public Config::GrpcMuxImpl, public StreamTrackedGrpcMux { public: - SotwGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node) - : Config::GrpcMuxImpl(grpc_mux_context, skip_subsequent_node) {} + SotwGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context, bool skip_subsequent_node, + std::function on_transport_established, + std::function on_transport_close) + : Config::GrpcMuxImpl(grpc_mux_context, skip_subsequent_node), + on_transport_established_(std::move(on_transport_established)), + on_transport_close_(std::move(on_transport_close)) {} ~SotwGrpcMuxImpl() override = default; void onStreamEstablished() override { + stream_connected_ = true; ++stream_generation_; Config::GrpcMuxImpl::onStreamEstablished(); + if (on_transport_established_) { + on_transport_established_(); + } + } + + void onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) override { + const bool was_connected = stream_connected_; + stream_connected_ = false; + Config::GrpcMuxImpl::onEstablishmentFailure(next_attempt_may_send_initial_resource_version); + if (was_connected && on_transport_close_) { + on_transport_close_(); + } } uint64_t streamGeneration() const override { return stream_generation_; } + bool streamConnected() const override { return stream_connected_; } private: uint64_t stream_generation_{0}; + bool stream_connected_{false}; + std::function on_transport_established_; + std::function on_transport_close_; }; class DeltaGrpcMuxImpl : public Config::NewGrpcMuxImpl, public StreamTrackedGrpcMux { public: - explicit DeltaGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context) - : Config::NewGrpcMuxImpl(grpc_mux_context) {} + explicit DeltaGrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context, + std::function on_transport_established, + std::function on_transport_close) + : Config::NewGrpcMuxImpl(grpc_mux_context), + on_transport_established_(std::move(on_transport_established)), + on_transport_close_(std::move(on_transport_close)) {} ~DeltaGrpcMuxImpl() override = default; void onStreamEstablished() override { + stream_connected_ = true; ++stream_generation_; Config::NewGrpcMuxImpl::onStreamEstablished(); + if (on_transport_established_) { + on_transport_established_(); + } + } + + void onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) override { + const bool was_connected = stream_connected_; + stream_connected_ = false; + Config::NewGrpcMuxImpl::onEstablishmentFailure(next_attempt_may_send_initial_resource_version); + if (was_connected && on_transport_close_) { + on_transport_close_(); + } } uint64_t streamGeneration() const override { return stream_generation_; } + bool streamConnected() const override { return stream_connected_; } private: uint64_t stream_generation_{0}; + bool stream_connected_{false}; + std::function on_transport_established_; + std::function on_transport_close_; }; // service RPC method fully qualified names. @@ -203,11 +247,27 @@ uint64_t grpcStreamGeneration(Config::Subscription* subscription) { return grpc_mux->streamGeneration(); } +bool grpcStreamConnected(Config::Subscription* subscription) { + auto* sub = dynamic_cast(subscription); + if (!sub) { + return false; + } + + auto* grpc_mux = dynamic_cast(sub->grpcMux().get()); + if (grpc_mux == nullptr) { + return false; + } + + return grpc_mux->streamConnected(); +} + std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, bool use_delta_xds, - std::chrono::milliseconds init_fetch_timeout) { + std::chrono::milliseconds init_fetch_timeout, + std::function on_transport_established, + std::function on_transport_close) { const envoy::config::core::v3::ConfigSource config_source = getCiliumXDSAPIConfig(use_delta_xds); const envoy::config::core::v3::ApiConfigSource& api_config_source = config_source.api_config_source(); @@ -249,10 +309,12 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte }; std::shared_ptr grpc_mux = - use_delta_xds ? std::static_pointer_cast( - std::make_shared(grpc_mux_context)) + use_delta_xds ? std::static_pointer_cast(std::make_shared( + grpc_mux_context, std::move(on_transport_established), + std::move(on_transport_close))) : std::static_pointer_cast(std::make_shared( - grpc_mux_context, api_config_source.set_node_on_first_message_only())); + grpc_mux_context, api_config_source.set_node_on_first_message_only(), + std::move(on_transport_established), std::move(on_transport_close))); return std::make_unique( grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index b66bec600..ba26b781f 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -20,7 +21,9 @@ std::unique_ptr subscribe(const std::string& type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, bool use_delta_xds = false, - std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); + std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0), + std::function on_transport_established = {}, + std::function on_transport_close = {}); // Returns a monotonic stream generation for Cilium subscriptions. // Value 0 is reserved for policy-map detection of the initial stream and may be returned for @@ -28,5 +31,8 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte // Non-gRPC subscriptions and subscriptions without stream tracking are treated as generation 1. uint64_t grpcStreamGeneration(Config::Subscription* subscription); +// Returns whether a tracked gRPC subscription currently has an established transport. +bool grpcStreamConnected(Config::Subscription* subscription); + } // namespace Cilium } // namespace Envoy diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 15b77947b..d13ef705c 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -353,6 +354,10 @@ class PolicyStreamState { using PolicyStreamStateSharedPtr = std::shared_ptr; using PolicyStreamStateConstSharedPtr = std::shared_ptr; +namespace { +constexpr absl::string_view WildcardResourceName = "*"; +} // namespace + class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, public Logger::Loggable, public std::enable_shared_from_this { @@ -361,23 +366,13 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, bool use_delta_xds); ~NetworkPolicyMapImpl() override; - void startSubscription() { - if (use_delta_xds_) { - subscription_ = subscribe("type.googleapis.com/cilium.NetworkPolicyResource", context_, - *npds_stats_scope_, *this, - std::make_shared( - ProtobufMessage::getNullValidationVisitor(), "name"), - use_delta_xds_); - } else { - subscription_ = - subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, *npds_stats_scope_, *this, - std::make_shared(), use_delta_xds_); - } - } + void subscribe(); // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { + void subscribe(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); + subscription_use_delta_xds_ = desired_use_delta_xds_; + subscription_connected_ = false; } // Config::SubscriptionCallbacks @@ -397,7 +392,15 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } - bool useDeltaXds() const { return use_delta_xds_; } + bool useDeltaXds() const { return desired_use_delta_xds_; } + + void setUseDeltaXds(bool use_delta_xds) { + desired_use_delta_xds_ = use_delta_xds; + if (!subscription_connected_ && subscription_ != nullptr) { + subscription_connected_ = grpcStreamConnected(subscription_.get()); + } + maybeRecreateSubscriptionInDesiredMode(); + } protected: uint64_t streamGeneration() const { @@ -432,6 +435,39 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, const PolicyStreamStateSharedPtr& policy_stream_state); private: + void startSubscription() { + ASSERT(subscription_ != nullptr); + if (subscription_use_delta_xds_) { + // NPDS always wants all resources, so use an explicit wildcard subscription in delta xDS. + subscription_->start({std::string(WildcardResourceName)}); + } else { + subscription_->start({}); + } + } + + void onSubscriptionTransportEstablished(uint64_t subscription_id) { + if (subscription_id != subscription_id_) { + return; + } + subscription_connected_ = true; + } + + void onSubscriptionTransportClosed(uint64_t subscription_id) { + if (subscription_id != subscription_id_) { + return; + } + subscription_connected_ = false; + maybeRecreateSubscriptionInDesiredMode(); + } + + void maybeRecreateSubscriptionInDesiredMode() { + if (subscription_ == nullptr || subscription_connected_ || + desired_use_delta_xds_ == subscription_use_delta_xds_) { + return; + } + subscribe(); + } + // Helpers for atomic swap of the policy map pointer. // // store() is only used for the initialization of the map during construction. @@ -479,10 +515,25 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void scheduleSelectorDeferredDeletion(DeferredDeletion&& deferred); void scheduleSelectorGCAndDeferredDeletion(uint64_t published_version, const PolicyMapSnapshot* old_policy_map = nullptr); + void startManagedSubscriptionForTest() { + subscription_should_start_ = true; + subscribe(); + } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + subscription_factory_for_test_ = std::move(factory); + } + void onSubscriptionConnectedForTest() { onSubscriptionTransportEstablished(subscription_id_); } + void onSubscriptionTransportCloseForTest() { onSubscriptionTransportClosed(subscription_id_); } + bool subscriptionUseDeltaXdsForTest() const { return subscription_use_delta_xds_; } + bool subscriptionConnectedForTest() const { return subscription_connected_; } static uint64_t instance_id_; - const bool use_delta_xds_; + bool desired_use_delta_xds_; + bool subscription_use_delta_xds_; + bool subscription_connected_{false}; + bool subscription_should_start_{false}; + uint64_t subscription_id_{0}; Server::Configuration::ServerFactoryContext& context_; std::atomic map_ptr_; @@ -502,6 +553,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, transport_factory_context_; std::unique_ptr subscription_; + NetworkPolicyMap::SubscriptionFactoryForTest subscription_factory_for_test_; // Test-only override used to simulate a restarted NPDS stream when the test subscription does // not expose a new underlying gRPC stream generation. uint64_t stream_generation_override_for_test_{0}; @@ -517,10 +569,6 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, uint64_t NetworkPolicyMapImpl::instance_id_ = 0; -namespace { -constexpr absl::string_view WildcardResourceName = "*"; -} // namespace - IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { auto ip = Network::Utility::parseInternetAddressNoThrow(ip_addr); @@ -2015,7 +2063,7 @@ NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& contex impl_ = std::make_shared(context, use_delta_xds); if (subscribe) { - impl_->startSubscription(); + impl_->subscribe(); } } @@ -2043,10 +2091,35 @@ bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { } bool NetworkPolicyMap::useDeltaXds() const { return impl_->useDeltaXds(); } +void NetworkPolicyMap::setUseDeltaXds(bool use_delta_xds) const { + impl_->setUseDeltaXds(use_delta_xds); +} void NetworkPolicyMap::startSubscriptionForTest( std::unique_ptr&& subscription) { - impl_->startSubscription(std::move(subscription)); + impl_->subscribe(std::move(subscription)); +} + +void NetworkPolicyMap::startManagedSubscriptionForTest() { + impl_->startManagedSubscriptionForTest(); +} + +void NetworkPolicyMap::setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory) { + impl_->setSubscriptionFactoryForTest(std::move(factory)); +} + +void NetworkPolicyMap::onSubscriptionConnectedForTest() { impl_->onSubscriptionConnectedForTest(); } + +void NetworkPolicyMap::onSubscriptionTransportCloseForTest() { + impl_->onSubscriptionTransportCloseForTest(); +} + +bool NetworkPolicyMap::subscriptionUseDeltaXdsForTest() const { + return impl_->subscriptionUseDeltaXdsForTest(); +} + +bool NetworkPolicyMap::subscriptionConnectedForTest() const { + return impl_->subscriptionConnectedForTest(); } Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksForTest() const { @@ -2073,18 +2146,15 @@ SelectorVersion NetworkPolicyMap::policySelectorVersionForTest(const PolicyInsta NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, bool use_delta_xds) - : use_delta_xds_(use_delta_xds), context_(context.serverFactoryContext()), map_ptr_(nullptr), + : desired_use_delta_xds_(use_delta_xds), subscription_use_delta_xds_(use_delta_xds), + context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), init_target_(fmt::format("Cilium Network Policy subscription start"), [this]() { - if (use_delta_xds_) { - // NPDS always wants all resources, so use an explicit wildcard subscription - // in delta xDS. - subscription_->start({std::string(WildcardResourceName)}); - } else { - subscription_->start({}); - } + // production subscription is allowed to start from now on + subscription_should_start_ = true; + startSubscription(); // Allow listener init to continue before network policy updates are received init_target_.ready(); }), @@ -2118,6 +2188,50 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { delete load(); } +void NetworkPolicyMapImpl::subscribe() { + subscription_connected_ = false; + subscription_use_delta_xds_ = desired_use_delta_xds_; + ++subscription_id_; + + if (subscription_factory_for_test_) { + subscription_ = subscription_factory_for_test_(subscription_use_delta_xds_); + if (subscription_should_start_) { + startSubscription(); + } + return; + } + + auto on_transport_close = [weak_this = weak_from_this(), id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionTransportClosed(id); + } + }; + auto on_transport_established = [weak_this = weak_from_this(), id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionTransportEstablished(id); + } + }; + + if (subscription_use_delta_xds_) { + subscription_ = Cilium::subscribe( + "type.googleapis.com/cilium.NetworkPolicyResource", context_, *npds_stats_scope_, *this, + std::make_shared(ProtobufMessage::getNullValidationVisitor(), + "name"), + subscription_use_delta_xds_, std::chrono::milliseconds(0), + std::move(on_transport_established), std::move(on_transport_close)); + } else { + subscription_ = + Cilium::subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, *npds_stats_scope_, + *this, std::make_shared(), + subscription_use_delta_xds_, std::chrono::milliseconds(0), + std::move(on_transport_established), std::move(on_transport_close)); + } + + if (subscription_should_start_) { + startSubscription(); + } +} + void NetworkPolicyMapImpl::reopenIpcache() { // Get ipcache singleton only if it was successfully created previously. // Cilium agent re-creates IP cache on restart, and the first accepted update on @@ -2255,6 +2369,7 @@ void NetworkPolicyMapImpl::scheduleSelectorGCAndDeferredDeletion( absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { + subscription_connected_ = true; auto stream_generation = streamGeneration(); const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", @@ -2329,6 +2444,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { + subscription_connected_ = true; auto stream_generation = streamGeneration(); const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); const auto& old_resource_map = resource_map_; diff --git a/cilium/network_policy.h b/cilium/network_policy.h index cd4b9900f..d8c4c5cab 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -193,12 +194,16 @@ class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: + using SubscriptionFactoryForTest = + std::function(bool use_delta_xds)>; + NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool subscribe = false, bool use_delta_xds = false); ~NetworkPolicyMap() override; bool exists(const std::string& endpoint_policy_name) const; bool useDeltaXds() const; + void setUseDeltaXds(bool use_delta_xds) const; const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; @@ -216,6 +221,12 @@ class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable&& subscription); + void startManagedSubscriptionForTest(); + void setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory); + void onSubscriptionConnectedForTest(); + void onSubscriptionTransportCloseForTest(); + bool subscriptionUseDeltaXdsForTest() const; + bool subscriptionConnectedForTest() const; Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index f638b47cf..b78315870 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -3,10 +3,12 @@ #include #include +#include #include #include #include #include +#include #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" @@ -32,6 +34,7 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" #include "cilium/network_policy.h" @@ -51,6 +54,34 @@ namespace Cilium { return secret_provider; \ })) +namespace { + +struct FakeSubscriptionState { + int start_calls_{0}; + std::vector> start_resources_; +}; + +class FakeSubscription : public Envoy::Config::Subscription { +public: + explicit FakeSubscription(std::shared_ptr state) + : state_(std::move(state)) {} + + void start(const absl::flat_hash_set& resource_names) override { + ++state_->start_calls_; + auto& started = state_->start_resources_.emplace_back(); + started.insert(started.end(), resource_names.begin(), resource_names.end()); + std::sort(started.begin(), started.end()); + } + + void updateResourceInterest(const absl::flat_hash_set&) override {} + void requestOnDemandUpdate(const absl::flat_hash_set&) override {} + +private: + std::shared_ptr state_; +}; + +} // namespace + class CiliumNetworkPolicyTest : public ::testing::Test { protected: CiliumNetworkPolicyTest() { @@ -251,6 +282,18 @@ class CiliumNetworkPolicyTest : public ::testing::Test { } void resetStreamForTest() { policy_map_->resetStreamForTest(); } + bool configuredUseDeltaXds() const { return policy_map_->useDeltaXds(); } + void setUseDeltaXds(bool use_delta_xds) const { policy_map_->setUseDeltaXds(use_delta_xds); } + void startManagedSubscriptionForTest() { policy_map_->startManagedSubscriptionForTest(); } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + policy_map_->setSubscriptionFactoryForTest(std::move(factory)); + } + void onSubscriptionConnectedForTest() { policy_map_->onSubscriptionConnectedForTest(); } + void onSubscriptionTransportCloseForTest() { policy_map_->onSubscriptionTransportCloseForTest(); } + bool subscriptionUseDeltaXdsForTest() const { + return policy_map_->subscriptionUseDeltaXdsForTest(); + } + bool subscriptionConnectedForTest() const { return policy_map_->subscriptionConnectedForTest(); } NiceMock factory_context_; NiceMock secret_manager_; @@ -268,6 +311,116 @@ TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { EXPECT_EQ("cilium.policy.updates_rejected", updatesRejectedStatName()); } +TEST_F(CiliumNetworkPolicyTest, ManagedSubscriptionColdStartUsesConfiguredSotwMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_FALSE(created_modes.front()); + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, ManagedSubscriptionColdStartUsesConfiguredDeltaMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_TRUE(created_modes.front()); + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_THAT(state->start_resources_.front(), testing::ElementsAre(std::string("*"))); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipOnHealthySubscriptionWaitsForTransportClose) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_TRUE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false)); + EXPECT_EQ(state->start_calls_, 1); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); +} + +TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipKeepsCurrentMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false)); + EXPECT_EQ(state->start_calls_, 1); +} + TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) { EXPECT_TRUE(subscriptionCallbacks().onConfigUpdate({}, "1").ok()); EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found From a58fddb0ada32b75084be291cb1133ca4da1f7b1 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 16 Apr 2026 16:19:35 +0200 Subject: [PATCH 09/14] policy: Add detail to NACK due to name/IP collision Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 169 +++++++++++++++++++++++++--- tests/cilium_network_policy_test.cc | 71 +++++++----- 2 files changed, 192 insertions(+), 48 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index d13ef705c..0399886f7 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -207,6 +207,8 @@ class ResourceMap : public absl::flat_hash_map { return it != end() ? &it->second : nullptr; } + std::string findPolicyResourceName(const std::shared_ptr& policy) const; + void replaceWith(std::vector>&& entries) { clear(); reserve(entries.size()); @@ -239,6 +241,11 @@ class ResourceMapOverlay { return base_ ? base_->findEntry(key) : nullptr; } + std::string findPolicyResourceName(const std::shared_ptr& policy) const; + + std::string describeExistingResourceKey(const std::string& key, + const PolicyMapSnapshot& policy_map) const; + SelectorHandle getSelectorHandleOrThrow(const std::string& selector) const { const auto* entry = findEntry(selector); if (entry == nullptr) { @@ -2013,6 +2020,107 @@ class PolicyInstanceImpl : public PolicyInstance { const PortNetworkPolicy egress_; }; +template std::string endpointIpsForLog(const EndpointIps& endpoint_ips) { + std::string formatted = "["; + bool first = true; + for (const auto& endpoint_ip : endpoint_ips) { + if (!first) { + formatted += ", "; + } + formatted += endpoint_ip; + first = false; + } + formatted += "]"; + return formatted; +} + +std::string describePolicyResourceForLog(absl::string_view resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource description requires a policy"); + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +std::string describePolicyResourceForLog(absl::string_view resource_name, + const cilium::NetworkPolicy& policy) { + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy.endpoint_id(), endpointIpsForLog(policy.endpoint_ips())); +} + +std::string +ResourceMap::findPolicyResourceName(const std::shared_ptr& policy) const { + if (policy == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : *this) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + return {}; +} + +std::string ResourceMapOverlay::findPolicyResourceName( + const std::shared_ptr& policy) const { + if (policy == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : upserts_) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + if (base_ == nullptr) { + return {}; + } + for (const auto& [resource_name, resource_key] : *base_) { + if (removed_.contains(resource_name)) { + continue; + } + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry != nullptr && policy_entry->policy == policy) { + return resource_name; + } + } + return {}; +} + +std::string +ResourceMapOverlay::describeExistingResourceKey(const std::string& key, + const PolicyMapSnapshot& policy_map) const { + const auto* entry = findEntry(key); + if (entry == nullptr) { + return fmt::format("resource key '{}'", key); + } + if (entry->selectorResourceEntry() != nullptr) { + return fmt::format("selector resource '{}'", key); + } + if (const auto* policy_entry = entry->policyResourceEntry(); + policy_entry != nullptr && policy_entry->policy != nullptr) { + return describePolicyResourceForLog(key, policy_entry->policy); + } + if (!entry->isPolicyEndpointIpEntry()) { + return fmt::format("resource key '{}'", key); + } + + auto policy_it = policy_map.find(key); + if (policy_it == policy_map.end()) { + return fmt::format("endpoint IP alias '{}'", key); + } + + const auto& policy = policy_it->second; + const auto resource_name = findPolicyResourceName(policy); + if (!resource_name.empty()) { + return fmt::format("endpoint IP alias '{}' owned by {}", key, + describePolicyResourceForLog(resource_name, policy)); + } + + return fmt::format("endpoint IP alias '{}' owned by endpoint_id {} with endpoint_ips {}", key, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + void ResourceMap::erasePolicyResource(PolicyMapSnapshot& policy_map, const std::string& resource_name, const std::shared_ptr& policy) { @@ -2505,25 +2613,29 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( try { const auto selector_update_version = selector_map_.prepareNextVersion(); - for (const auto& removed_resource : removed_resources) { - ENVOY_LOG(trace, "Cilium removing network policy resource {}", removed_resource); - const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + for (const auto& resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing network policy selector resource {}", resource); + const auto* resource_entry = pending_resource_map.findEntry(resource); if (resource_entry == nullptr) { + ENVOY_LOG( + debug, + "NetworkPolicy delta removed selector resource name '{}' not found from resource map", + resource); continue; } if (resource_entry->isPolicyEndpointIpEntry()) { - throw EnvoyException(fmt::format( - "Network Policy delta removed resource '{}' is a policy endpoint IP alias, " - "not a resource name", - removed_resource)); + throw EnvoyException(fmt::format("NetworkPolicy delta removed selector resource name " + "'{}' is a policy endpoint IP alias, " + "not a resource name", + resource)); } if (resource_entry->policyResourceEntry()) { - throw EnvoyException( - fmt::format("Network Policy delta removed resource '{}' refers to a policy resource", - removed_resource)); + throw EnvoyException(fmt::format( + "NetworkPolicy delta removed selector resource name '{}' refers to a policy resource", + resource)); } - selector_map_.clear(removed_resource); - pending_resource_map.erase(removed_resource); + selector_map_.clear(resource); + pending_resource_map.erase(resource); } for (const auto& resource : added_resources) { @@ -2552,7 +2664,11 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (!pending_resource_map.emplace(resource_name, ResourceKey::selectorResource(selector_handle))) { throw EnvoyException(fmt::format( - "Network Policy delta update has duplicate resource key '{}'", resource_name)); + "Network Policy delta selector update for version {} has duplicate resource key " + "'{}' on an old stream: " + "incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, resource_name, + pending_resource_map.describeExistingResourceKey(resource_name, *load()))); } break; } @@ -2658,7 +2774,11 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (!pending_resource_map.emplace(resource_name, ResourceKey::selectorResource(selector_handle))) { throw EnvoyException(fmt::format( - "Network Policy delta update has duplicate resource key '{}'", resource_name)); + "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", resource_name, + pending_resource_map.describeExistingResourceKey(resource_name, new_policy_map))); } } @@ -2688,17 +2808,32 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( old_resource_map, &pending_resource_map); if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { throw EnvoyException(fmt::format( - "Network Policy delta update has duplicate resource key '{}'", resource_name)); + "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(resource_name, new_policy_map))); } for (const auto& endpoint_ip : config.endpoint_ips()) { ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp())) { throw EnvoyException(fmt::format( - "Network Policy delta update has duplicate resource key '{}'", endpoint_ip)); + "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip, new_policy_map))); } if (!new_policy_map.emplace(endpoint_ip, policy).second) { throw EnvoyException(fmt::format( - "Network Policy delta update has duplicate resource key '{}'", endpoint_ip)); + "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "stream: " + "incoming {} collides with existing {}", + system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip, new_policy_map))); } } break; diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index b78315870..18173c53b 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -860,7 +860,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBe EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "10.1.2.3" version: "1" @@ -869,8 +870,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBe selector: remote_identities: [ 44 ] )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.3'"); + EnvoyException, + R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); @@ -1163,7 +1164,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitesp } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" resources: - name: "shared-name" version: "1" @@ -1182,8 +1184,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInS - "10.1.2.4" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key 'shared-name'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { @@ -1203,7 +1205,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMat } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdate) { - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" resources: - name: "policy-a" version: "1" @@ -1222,12 +1225,13 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdat - "10.1.2.3" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.3'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" resources: - name: "10.1.2.4" version: "1" @@ -1246,12 +1250,13 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollision - "10.1.2.4" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.4'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" resources: - name: "shared-selector" version: "1" @@ -1266,9 +1271,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesI selector: remote_identities: [ 46, 47 ] )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key " - "'shared-selector'"); + EnvoyException, + R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { @@ -1343,7 +1347,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistin endpoint_id: 42 )EOF")); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "policy-b" version: "1" @@ -1354,8 +1359,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistin - "10.1.2.3" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.3'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1372,7 +1377,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 42 )EOF")); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "10.1.2.3" version: "1" @@ -1383,8 +1389,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, - "10.1.2.4" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.3'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1401,7 +1407,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 42 )EOF")); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "policy-b" version: "1" @@ -1412,8 +1419,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, - "10.1.2.4" endpoint_id: 43 )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.4'"); + EnvoyException, + R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1430,7 +1437,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 42 )EOF")); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "shared-name" version: "1" @@ -1439,8 +1447,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, selector: remote_identities: [ 43 ] )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key 'shared-name'"); + EnvoyException, + R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1457,7 +1465,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 42 )EOF")); - EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" resources: - name: "10.1.2.3" version: "1" @@ -1466,8 +1475,8 @@ TEST_F(CiliumNetworkPolicyDeltaTest, selector: remote_identities: [ 43 ] )EOF"), - EnvoyException, - "Network Policy delta update has duplicate resource key '10.1.2.3'"); + EnvoyException, + R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { From a81160220be5ce8af7aad4d5f1762a646baf83ed Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Fri, 17 Apr 2026 22:08:56 +0200 Subject: [PATCH 10/14] policy: clear resource map on a new stream Clear the resource map on a first update on a new stream. This fixes NACK cases where further updates on the stream would have IP collisions with resources that were kept from the previous stream. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 11 ++++ tests/cilium_network_policy_test.cc | 96 +++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 0399886f7..3cbf2bc4d 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -2479,6 +2479,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::string& version_info) { subscription_connected_ = true; auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", instance_id_, resources.size(), version_info); @@ -2554,8 +2556,12 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::string& system_version_info) { subscription_connected_ = true; auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); const auto& old_resource_map = resource_map_; + + // first find if this is a selector-only update bool updates_policies = false; bool updates_selectors = false; for (const auto& removed_resource : removed_resources) { @@ -2590,6 +2596,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( break; } } + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " "version: {}, updates_selectors: {}, updates_policies: {}", @@ -2853,6 +2860,10 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( removeInitManager(); installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), policy_stream_state); + // do not carry over any resources from an old stream + if (is_new_stream) { + resource_map_.clear(); + } std::move(pending_resource_map).applyTo(resource_map_); return absl::OkStatus(); diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index 18173c53b..ce0c3979a 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -1681,6 +1681,102 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamReplacesStateWithFullSnapshot EXPECT_FALSE(policy_map_->exists("10.2.3.4")); } +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamClearsStaleResourceNamesFromResourceMap) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46 ] +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "3" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 9090 + rules: + - selectors: [ "policy-43" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 46, 9090)); +} + TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelectorSnapshot) { EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" resources: From bbbc2469f0fef12317476f01f26440a48610dfe8 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sat, 18 Apr 2026 11:04:08 +0200 Subject: [PATCH 11/14] policy: clean-up Do not use the default decoder as NPRDS does not carry a name inside the NetworkPolicyResource type. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 75 ++++++++++++++--------------- cilium/network_policy.h | 32 ++++++++++-- tests/cilium_network_policy_test.cc | 3 +- 3 files changed, 66 insertions(+), 44 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 3cbf2bc4d..1afa4d2c3 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -45,7 +45,6 @@ #include "source/common/init/target_impl.h" #include "source/common/init/watcher_impl.h" #include "source/common/network/utility.h" -#include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" #include "source/server/transport_socket_config_impl.h" @@ -327,8 +326,10 @@ class ResourceMapOverlay { }; // helper for validating resource names. -void validateResourceNameHasNoWhitespace(absl::string_view resource_name, - absl::string_view subject) { +void validateResourceName(absl::string_view resource_name, absl::string_view subject) { + if (resource_name.empty()) { + throw EnvoyException(fmt::format("{} must not be empty", subject)); + } if (std::ranges::any_of(resource_name, [](unsigned char c) { return absl::ascii_isspace(c); })) { throw EnvoyException( fmt::format("{} '{}' must not contain whitespace", subject, resource_name)); @@ -431,8 +432,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, std::shared_ptr createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, const PolicyStreamStateConstSharedPtr& policy_stream_state, - const ResourceMap& old_resource_map, - const ResourceMapOverlay* selector_resource_map); + const ResourceMap& resource_map, + const ResourceMapOverlay* pending_resource_map); SelectorHandle createOrReuseSelector(const std::string& resource_name, const cilium::Selector& config, uint64_t update_version); @@ -939,7 +940,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, const cilium::PortNetworkPolicyRule& rule, - const ResourceMapOverlay* selector_resource_map) + const ResourceMapOverlay* resource_map) : name_(rule.name()), verdict_(rule.pass_precedence() ? RuleVerdict::Pass : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), @@ -950,7 +951,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } - if (selector_resource_map) { + if (resource_map) { if (rule.remote_policies_size()) { throw EnvoyException( "Delta Network Policy rule must use selectors instead of remote_policies"); @@ -959,7 +960,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { for (const auto& selector : rule.selectors()) { ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} selector {} by rule: {}", verdict_, selector, name_); - selectors_.emplace_back(selector_resource_map->getSelectorHandleOrThrow(selector)); + selectors_.emplace_back(resource_map->getSelectorHandleOrThrow(selector)); } } else { if (rule.selectors_size()) { @@ -1301,14 +1302,14 @@ class PortNetworkPolicyRules : public Logger::Loggable { // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, const Protobuf::RepeatedPtrField& rules, - const ResourceMapOverlay* selector_resource_map) { + const ResourceMapOverlay* resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { rules_.emplace(rules_.begin(), - std::make_shared(parent, it, selector_resource_map)); + std::make_shared(parent, it, resource_map)); updateFor(rules_.front()); } initialized_ = true; @@ -1715,7 +1716,7 @@ class PortNetworkPolicy : public Logger::Loggable { public: PortNetworkPolicy(const NetworkPolicyMapImpl& parent, const Protobuf::RepeatedPtrField& rules, - const ResourceMapOverlay* selector_resource_map) { + const ResourceMapOverlay* resource_map) { for (const auto& rule : rules) { // Only TCP supported for HTTP if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { @@ -1874,10 +1875,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules(), selector_resource_map); + rules.prepend(parent, rule.rules(), resource_map); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules(), selector_resource_map); + rules.append(parent, rule.rules(), resource_map); } } } else { @@ -1958,11 +1959,11 @@ class PolicyInstanceImpl : public PolicyInstance { PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, const cilium::NetworkPolicy& proto, const PolicyStreamStateConstSharedPtr& policy_stream_state, - const ResourceMapOverlay* selector_resource_map) + const ResourceMapOverlay* resource_map) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), parent_(parent), policy_stream_state_(policy_stream_state), - ingress_(parent, policy_proto_.ingress_per_port_policies(), selector_resource_map), - egress_(parent, policy_proto_.egress_per_port_policies(), selector_resource_map) {} + ingress_(parent, policy_proto_.ingress_per_port_policies(), resource_map), + egress_(parent, policy_proto_.egress_per_port_policies(), resource_map) {} bool allowed(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, Envoy::Http::RequestHeaderMap& headers, @@ -2323,10 +2324,9 @@ void NetworkPolicyMapImpl::subscribe() { if (subscription_use_delta_xds_) { subscription_ = Cilium::subscribe( "type.googleapis.com/cilium.NetworkPolicyResource", context_, *npds_stats_scope_, *this, - std::make_shared(ProtobufMessage::getNullValidationVisitor(), - "name"), - subscription_use_delta_xds_, std::chrono::milliseconds(0), - std::move(on_transport_established), std::move(on_transport_close)); + std::make_shared(), subscription_use_delta_xds_, + std::chrono::milliseconds(0), std::move(on_transport_established), + std::move(on_transport_close)); } else { subscription_ = Cilium::subscribe("type.googleapis.com/cilium.NetworkPolicy", context_, *npds_stats_scope_, @@ -2353,27 +2353,27 @@ void NetworkPolicyMapImpl::reopenIpcache() { std::shared_ptr NetworkPolicyMapImpl::createOrReusePolicy( const std::string& resource_name, const cilium::NetworkPolicy& config, - const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& old_resource_map, - const ResourceMapOverlay* selector_resource_map) { + const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& resource_map, + const ResourceMapOverlay* pending_resource_map) { const uint64_t new_hash = MessageUtil::hash(config); - auto it = old_resource_map.find(resource_name); - if (it != old_resource_map.cend()) { + auto it = resource_map.find(resource_name); + if (it != resource_map.cend()) { const auto* old_policy_entry = it->second.policyResourceEntry(); if (old_policy_entry == nullptr) { return std::make_shared(*this, new_hash, config, - policy_stream_state, selector_resource_map); + policy_stream_state, pending_resource_map); } const auto& old_policy = old_policy_entry->policy; if (old_policy && old_policy->hash_ == new_hash && Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config) && - !(selector_resource_map && policyUsesSelectors(config))) { + !(pending_resource_map && policyUsesSelectors(config))) { ENVOY_LOG(trace, "New policy is equal to old one, not updating."); return old_policy; } } return std::make_shared(*this, new_hash, config, policy_stream_state, - selector_resource_map); + pending_resource_map); } SelectorHandle NetworkPolicyMapImpl::createOrReuseSelector(const std::string& resource_name, @@ -2503,7 +2503,6 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // SDS secrets will use this! transport_factory_context_->setInitManager(version_init_manager); - const auto& old_resource_map = resource_map_; const auto policy_stream_state = is_new_stream ? std::make_shared(stream_generation, selector_map_.getVersion()) @@ -2514,7 +2513,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( for (const auto& resource : resources) { const auto& config = dynamic_cast(resource.get().resource()); const std::string& resource_name = resource.get().name(); - validateResourceNameHasNoWhitespace(resource_name, "Network Policy resource name"); + validateResourceName(resource_name, "Network Policy resource name"); if (config.endpoint_ips().empty()) { throw EnvoyException("Network Policy has no endpoint ips"); } @@ -2523,8 +2522,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( "version {}", config.endpoint_id(), config.endpoint_ips()[0], version_info); - auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, - old_resource_map, nullptr); + auto policy = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map_, nullptr); if (!resource_name.empty()) { resource_entries.emplace_back(resource_name, ResourceKey::policyResource(policy)); } @@ -2559,16 +2558,14 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // policy_stream_state_ gets updated on first successful update, // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); - const auto& old_resource_map = resource_map_; // first find if this is a selector-only update bool updates_policies = false; bool updates_selectors = false; for (const auto& removed_resource : removed_resources) { - validateResourceNameHasNoWhitespace(removed_resource, - "Network Policy delta removed resource name"); - auto resource_it = old_resource_map.find(removed_resource); - if (resource_it == old_resource_map.end()) { + validateResourceName(removed_resource, "Network Policy delta removed resource name"); + auto resource_it = resource_map_.find(removed_resource); + if (resource_it == resource_map_.end()) { continue; } if (resource_it->second.selectorResourceEntry()) { @@ -2584,7 +2581,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (resource_name.empty()) { throw EnvoyException("Network Policy delta resource has no name"); } - validateResourceNameHasNoWhitespace(resource_name, "Network Policy delta resource name"); + validateResourceName(resource_name, "Network Policy delta resource name"); switch (typed_resource.resource_case()) { case cilium::NetworkPolicyResource::kPolicy: updates_policies = true; @@ -2811,8 +2808,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( resource_name, config.endpoint_id(), config.endpoint_ips()[0], system_version_info); - auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, - old_resource_map, &pending_resource_map); + auto policy = createOrReusePolicy(resource_name, config, policy_stream_state, resource_map_, + &pending_resource_map); if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { throw EnvoyException(fmt::format( "Network Policy delta update for version {} has duplicate resource key '{}' on {} " diff --git a/cilium/network_policy.h b/cilium/network_policy.h index d8c4c5cab..8d4685c84 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -7,6 +7,7 @@ #include #include +#include "envoy/common/exception.h" #include "envoy/common/pure.h" #include "envoy/common/regex.h" #include "envoy/config/core/v3/base.pb.h" @@ -24,7 +25,6 @@ #include "source/common/common/logger.h" #include "source/common/common/macros.h" #include "source/common/common/thread.h" -#include "source/common/config/opaque_resource_decoder_impl.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" @@ -170,8 +170,34 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { ProtobufMessage::ValidationVisitor& validation_visitor_; }; -using NetworkPolicyResourceDecoder = - Envoy::Config::OpaqueResourceDecoderImpl; +// cilium::NetworkPolicyResource does not carry a resource name, but relies on the +// DeltaDiscoveryRespons Resource wrapper to have the name. Hence can not use +// Envoy::Config::OpaqueResourceDecoderImpl +class NetworkPolicyResourceDecoder : public Envoy::Config::OpaqueResourceDecoder { +public: + NetworkPolicyResourceDecoder() + : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} + + // Config::OpaqueResourceDecoder + ProtobufTypes::MessagePtr decodeResource(const Protobuf::Any& resource) override { + auto typed_message = std::make_unique(); + // If the Any is a synthetic empty message (e.g. because the resource field + // was not set in Resource, this might be empty, so we shouldn't decode. + if (!resource.type_url().empty()) { + MessageUtil::anyConvertAndValidate(resource, *typed_message, + validation_visitor_); + } + return typed_message; + } + + std::string resourceName(const Protobuf::Message&) override { + throw EnvoyException( + "NetworkPolicyResource does not carry a name and must be wrapped in Resource"); + } + +private: + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; /** * All Cilium L7 filter stats. @see stats_macros.h diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index ce0c3979a..5c954e767 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -135,8 +135,7 @@ class CiliumNetworkPolicyTest : public ::testing::Test { std::string deltaUpdateFromYaml(const std::string& config) { envoy::service::discovery::v3::DeltaDiscoveryResponse message; MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); - NetworkPolicyResourceDecoder network_policy_resource_decoder( - ProtobufMessage::getNullValidationVisitor(), "name"); + NetworkPolicyResourceDecoder network_policy_resource_decoder; auto decoded_resources = std::make_unique(); for (const auto& resource : message.resources()) { decoded_resources->pushBack( From 18261d2c12bc29736e56bbd4d1af65e9f0b85b78 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sat, 18 Apr 2026 20:32:00 +0200 Subject: [PATCH 12/14] policy: Fix stream generation accounting Stream generation accounting has to be shared between NPDS and NPRDS streams, so that the handoff works as designed, but no other xDS protocols (e.g., NPHDS) should interfere with the stream generation accounting. Solve this by defining the stream generation number as a static member of NetworkPolicyMapImpl and updating it from the already established transport connected/closed callbacks. Adjust tests to work with the new shape where the generation numbers do not start from 1 for each NetworkPolicyMapImpl instance, but increase monotonically for each established NPDS/NPRDS stream. Signed-off-by: Jarno Rajahalme --- cilium/grpc_subscription.cc | 92 ++++++++++------------------- cilium/grpc_subscription.h | 6 -- cilium/network_policy.cc | 15 ++--- tests/cilium_network_policy_test.cc | 16 +++-- 4 files changed, 47 insertions(+), 82 deletions(-) diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index 3e8ea4956..a3fad38fc 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -43,13 +43,34 @@ namespace Cilium { namespace { -constexpr uint64_t FirstStreamGeneration = 1; - class StreamTrackedGrpcMux { public: + StreamTrackedGrpcMux(std::function on_transport_established, + std::function on_transport_close) + : on_transport_established_(std::move(on_transport_established)), + on_transport_close_(std::move(on_transport_close)) {} + virtual ~StreamTrackedGrpcMux() = default; - virtual uint64_t streamGeneration() const = 0; - virtual bool streamConnected() const = 0; + + bool streamConnected() const { return stream_connected_; } + + void onStreamEstablished() { + stream_connected_ = true; + on_transport_established_(); + } + + void onEstablishmentFailure() { + const bool was_connected = stream_connected_; + stream_connected_ = false; + if (was_connected) { + on_transport_close_(); + } + } + +private: + bool stream_connected_{false}; + std::function on_transport_established_; + std::function on_transport_close_; }; class SotwGrpcMuxImpl : public Config::GrpcMuxImpl, public StreamTrackedGrpcMux { @@ -58,37 +79,18 @@ class SotwGrpcMuxImpl : public Config::GrpcMuxImpl, public StreamTrackedGrpcMux std::function on_transport_established, std::function on_transport_close) : Config::GrpcMuxImpl(grpc_mux_context, skip_subsequent_node), - on_transport_established_(std::move(on_transport_established)), - on_transport_close_(std::move(on_transport_close)) {} - + StreamTrackedGrpcMux(std::move(on_transport_established), std::move(on_transport_close)) {} ~SotwGrpcMuxImpl() override = default; void onStreamEstablished() override { - stream_connected_ = true; - ++stream_generation_; Config::GrpcMuxImpl::onStreamEstablished(); - if (on_transport_established_) { - on_transport_established_(); - } + StreamTrackedGrpcMux::onStreamEstablished(); } void onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) override { - const bool was_connected = stream_connected_; - stream_connected_ = false; Config::GrpcMuxImpl::onEstablishmentFailure(next_attempt_may_send_initial_resource_version); - if (was_connected && on_transport_close_) { - on_transport_close_(); - } + StreamTrackedGrpcMux::onEstablishmentFailure(); } - - uint64_t streamGeneration() const override { return stream_generation_; } - bool streamConnected() const override { return stream_connected_; } - -private: - uint64_t stream_generation_{0}; - bool stream_connected_{false}; - std::function on_transport_established_; - std::function on_transport_close_; }; class DeltaGrpcMuxImpl : public Config::NewGrpcMuxImpl, public StreamTrackedGrpcMux { @@ -97,37 +99,19 @@ class DeltaGrpcMuxImpl : public Config::NewGrpcMuxImpl, public StreamTrackedGrpc std::function on_transport_established, std::function on_transport_close) : Config::NewGrpcMuxImpl(grpc_mux_context), - on_transport_established_(std::move(on_transport_established)), - on_transport_close_(std::move(on_transport_close)) {} + StreamTrackedGrpcMux(std::move(on_transport_established), std::move(on_transport_close)) {} ~DeltaGrpcMuxImpl() override = default; void onStreamEstablished() override { - stream_connected_ = true; - ++stream_generation_; Config::NewGrpcMuxImpl::onStreamEstablished(); - if (on_transport_established_) { - on_transport_established_(); - } + StreamTrackedGrpcMux::onStreamEstablished(); } void onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) override { - const bool was_connected = stream_connected_; - stream_connected_ = false; Config::NewGrpcMuxImpl::onEstablishmentFailure(next_attempt_may_send_initial_resource_version); - if (was_connected && on_transport_close_) { - on_transport_close_(); - } + StreamTrackedGrpcMux::onEstablishmentFailure(); } - - uint64_t streamGeneration() const override { return stream_generation_; } - bool streamConnected() const override { return stream_connected_; } - -private: - uint64_t stream_generation_{0}; - bool stream_connected_{false}; - std::function on_transport_established_; - std::function on_transport_close_; }; // service RPC method fully qualified names. @@ -233,20 +217,6 @@ envoy::config::core::v3::ConfigSource getCiliumXDSAPIConfig(bool use_delta_xds = envoy::config::core::v3::ConfigSource cilium_xds_api_config = getCiliumXDSAPIConfig(); -uint64_t grpcStreamGeneration(Config::Subscription* subscription) { - auto* sub = dynamic_cast(subscription); - if (!sub) { - return FirstStreamGeneration; - } - - auto* grpc_mux = dynamic_cast(sub->grpcMux().get()); - if (grpc_mux == nullptr) { - return FirstStreamGeneration; - } - - return grpc_mux->streamGeneration(); -} - bool grpcStreamConnected(Config::Subscription* subscription) { auto* sub = dynamic_cast(subscription); if (!sub) { diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index ba26b781f..24c639a60 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -25,12 +25,6 @@ subscribe(const std::string& type_url, Server::Configuration::CommonFactoryConte std::function on_transport_established = {}, std::function on_transport_close = {}); -// Returns a monotonic stream generation for Cilium subscriptions. -// Value 0 is reserved for policy-map detection of the initial stream and may be returned for -// tracked gRPC subscriptions before any stream has been established. -// Non-gRPC subscriptions and subscriptions without stream tracking are treated as generation 1. -uint64_t grpcStreamGeneration(Config::Subscription* subscription); - // Returns whether a tracked gRPC subscription currently has an established transport. bool grpcStreamConnected(Config::Subscription* subscription); diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 1afa4d2c3..34e762480 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -411,12 +411,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } protected: - uint64_t streamGeneration() const { - return stream_generation_override_for_test_ != 0 ? stream_generation_override_for_test_ - : grpcStreamGeneration(subscription_.get()); - } - - void resetStreamForTest() { stream_generation_override_for_test_ = streamGeneration() + 1; } + uint64_t streamGeneration() const { return subscription_stream_generation_; } + void resetStreamForTest() { subscription_stream_generation_++; } // run the given function after all the threads have scheduled void runAfterAllThreads(std::function cb) const { @@ -454,6 +450,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } void onSubscriptionTransportEstablished(uint64_t subscription_id) { + ++subscription_stream_generation_; + if (subscription_id != subscription_id_) { return; } @@ -561,10 +559,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, transport_factory_context_; std::unique_ptr subscription_; + static uint64_t subscription_stream_generation_; NetworkPolicyMap::SubscriptionFactoryForTest subscription_factory_for_test_; - // Test-only override used to simulate a restarted NPDS stream when the test subscription does - // not expose a new underlying gRPC stream generation. - uint64_t stream_generation_override_for_test_{0}; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -576,6 +572,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, }; uint64_t NetworkPolicyMapImpl::instance_id_ = 0; +uint64_t NetworkPolicyMapImpl::subscription_stream_generation_ = 1; IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index 5c954e767..0fe92abc4 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -1801,8 +1801,10 @@ TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelec const auto old_policy = policyInstanceShared("10.1.2.3"); ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); - EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); EXPECT_EQ(1, selectorVersionForTest(*old_policy)); EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" @@ -1815,7 +1817,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelec remote_identities: [ 44 ] )EOF")); - EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); EXPECT_EQ(2, selectorVersionForTest(*old_policy)); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); @@ -1852,8 +1854,10 @@ TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorS const auto old_policy = policyInstanceShared("10.1.2.3"); ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); - EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); EXPECT_EQ(1, selectorVersionForTest(*old_policy)); EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" @@ -1866,7 +1870,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorS remote_identities: [ 45 ] )EOF")); - EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); EXPECT_EQ(2, selectorVersionForTest(*old_policy)); resetStreamForTest(); @@ -1903,9 +1907,9 @@ TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorS ASSERT_NE(nullptr, new_policy); EXPECT_NE(old_policy.get(), new_policy.get()); - EXPECT_EQ(1, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); EXPECT_EQ(2, selectorVersionForTest(*old_policy)); - EXPECT_EQ(2, selectorStreamGenerationForTest(*new_policy)); + EXPECT_EQ(initial_stream_generation + 1, selectorStreamGenerationForTest(*new_policy)); EXPECT_EQ(3, selectorVersionForTest(*new_policy)); EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); From e090d1d2359273763eb2cd93a6123dc6fbe239a3 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sun, 19 Apr 2026 07:46:47 +0200 Subject: [PATCH 13/14] policy: clean up logs and error messages Use NetworkPolicyResource and NetworkPolicy consistently for the two typeURLs to make grepping logs more meaningful. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 81 +++++++++++++++-------------- tests/cilium_network_policy_test.cc | 32 ++++++------ 2 files changed, 57 insertions(+), 56 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 34e762480..056fdfbd4 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -249,12 +249,12 @@ class ResourceMapOverlay { const auto* entry = findEntry(selector); if (entry == nullptr) { throw EnvoyException(fmt::format( - "Delta Network Policy rule references missing selector resource '{}'", selector)); + "NetworkPolicyResource rule references missing selector resource '{}'", selector)); } const auto* selector_entry = entry->selectorResourceEntry(); if (selector_entry == nullptr || selector_entry->handle == nullptr) { throw EnvoyException( - fmt::format("Delta Network Policy rule references non-selector resource '{}'", selector)); + fmt::format("NetworkPolicyResource rule references non-selector resource '{}'", selector)); } return selector_entry->handle; } @@ -951,7 +951,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { if (resource_map) { if (rule.remote_policies_size()) { throw EnvoyException( - "Delta Network Policy rule must use selectors instead of remote_policies"); + "NetworkPolicyResource rule must use selectors instead of remote_policies"); } selectors_.reserve(rule.selectors_size()); for (const auto& selector : rule.selectors()) { @@ -961,7 +961,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } else { if (rule.selectors_size()) { - throw EnvoyException("State-of-the-world Network Policy rule must not use selectors"); + throw EnvoyException("NetworkPolicy rule must not use selectors"); } for (const auto remote : rule.remote_policies()) { ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, @@ -1594,7 +1594,7 @@ struct PortRangeCompare { // PolicySnapshot is keyed by port ranges, and contains a list of PortNetworkPolicyRules's // applicable to this range. A list is needed as rules may come from multiple sources (e.g., -// resulting from use of named ports and numbered ports in Cilium Network Policy at the same time). +// resulting from use of named ports and numbered ports in Cilium NetworkPolicy at the same time). class PolicySnapshot : public absl::btree_map { public: using absl::btree_map::btree_map; @@ -2256,7 +2256,7 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl(Server::Configuration::FactoryContext context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), - init_target_(fmt::format("Cilium Network Policy subscription start"), + init_target_(fmt::format("Cilium NetworkPolicy subscription start"), [this]() { // production subscription is allowed to start from now on subscription_should_start_ = true; @@ -2479,8 +2479,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // policy_stream_state_ gets updated on first successful update, // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); - ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", - instance_id_, resources.size(), version_info); + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", + instance_id_, resources.size(), version_info, stream_generation); stats_.updates_total_.inc(); // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, @@ -2488,7 +2488,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. if (is_new_stream) { - ENVOY_LOG(info, "New NetworkPolicy stream"); + ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); reopenIpcache(); } @@ -2510,12 +2510,12 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( for (const auto& resource : resources) { const auto& config = dynamic_cast(resource.get().resource()); const std::string& resource_name = resource.get().name(); - validateResourceName(resource_name, "Network Policy resource name"); + validateResourceName(resource_name, "NetworkPolicy resource name"); if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); + throw EnvoyException("NetworkPolicy has no endpoint ips"); } ENVOY_LOG(debug, - "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " + "Received NetworkPolicy for endpoint {}, endpoint_ip {} in onConfigUpdate() " "version {}", config.endpoint_id(), config.endpoint_ips()[0], version_info); @@ -2560,7 +2560,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( bool updates_policies = false; bool updates_selectors = false; for (const auto& removed_resource : removed_resources) { - validateResourceName(removed_resource, "Network Policy delta removed resource name"); + validateResourceName(removed_resource, "NetworkPolicyResource removed resource name"); auto resource_it = resource_map_.find(removed_resource); if (resource_it == resource_map_.end()) { continue; @@ -2576,9 +2576,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( dynamic_cast(resource.get().resource()); const std::string& resource_name = resource.get().name(); if (resource_name.empty()) { - throw EnvoyException("Network Policy delta resource has no name"); + throw EnvoyException("NetworkPolicyResource has no name"); } - validateResourceName(resource_name, "Network Policy delta resource name"); + validateResourceName(resource_name, "NetworkPolicyResource added resource name"); switch (typed_resource.resource_case()) { case cilium::NetworkPolicyResource::kPolicy: updates_policies = true; @@ -2593,9 +2593,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " - "version: {}, updates_selectors: {}, updates_policies: {}", + "version: {}, stream {}, updates_selectors: {}, updates_policies: {}", instance_id_, added_resources.size(), removed_resources.size(), system_version_info, - updates_selectors, updates_policies); + stream_generation, updates_selectors, updates_policies); stats_.updates_total_.inc(); // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, @@ -2603,7 +2603,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. if (is_new_stream) { - ENVOY_LOG(info, "New NetworkPolicy stream"); + ENVOY_LOG(info, "New NetworkPolicyResource stream {}", stream_generation); reopenIpcache(); } removeInitManager(); @@ -2615,24 +2615,24 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const auto selector_update_version = selector_map_.prepareNextVersion(); for (const auto& resource : removed_resources) { - ENVOY_LOG(trace, "Cilium removing network policy selector resource {}", resource); + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource selector {}", resource); const auto* resource_entry = pending_resource_map.findEntry(resource); if (resource_entry == nullptr) { ENVOY_LOG( debug, - "NetworkPolicy delta removed selector resource name '{}' not found from resource map", + "NetworkPolicyResource removed selector name '{}' not found from resource map", resource); continue; } if (resource_entry->isPolicyEndpointIpEntry()) { - throw EnvoyException(fmt::format("NetworkPolicy delta removed selector resource name " + throw EnvoyException(fmt::format("NetworkPolicyResource removed selector name " "'{}' is a policy endpoint IP alias, " "not a resource name", resource)); } if (resource_entry->policyResourceEntry()) { throw EnvoyException(fmt::format( - "NetworkPolicy delta removed selector resource name '{}' refers to a policy resource", + "NetworkPolicyResource removed selector name '{}' refers to a policy resource", resource)); } selector_map_.clear(resource); @@ -2657,7 +2657,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( switch (typed_resource.resource_case()) { case cilium::NetworkPolicyResource::kSelector: { ENVOY_LOG(debug, - "Received delta Network Policy selector resource {} in onConfigUpdate() " + "Received NetworkPolicyResource selector {} in onConfigUpdate() " "version {}", resource_name, system_version_info); auto selector_handle = createOrReuseSelector(resource_name, typed_resource.selector(), @@ -2665,7 +2665,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (!pending_resource_map.emplace(resource_name, ResourceKey::selectorResource(selector_handle))) { throw EnvoyException(fmt::format( - "Network Policy delta selector update for version {} has duplicate resource key " + "NetworkPolicyResource selector update for version {} has duplicate resource key " "'{}' on an old stream: " "incoming selector resource '{}' collides with existing {}", system_version_info, resource_name, resource_name, @@ -2674,14 +2674,14 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( break; } case cilium::NetworkPolicyResource::kPolicy: - IS_ENVOY_BUG("Selector-only delta Network Policy update unexpectedly included a policy"); + IS_ENVOY_BUG("Selector-only NetworkPolicyResource update unexpectedly included a policy"); break; case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: - throw EnvoyException("Network Policy delta resource has no payload"); + throw EnvoyException("NetworkPolicyResource has no payload"); } } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy delta update for version {} failed: {}", system_version_info, + ENVOY_LOG(warn, "NetworkPolicyResource update for version {} failed: {}", system_version_info, e.what()); stats_.updates_rejected_.inc(); scheduleSelectorDeferredDeletion(selector_map_.revert()); @@ -2717,7 +2717,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( const auto selector_update_version = selector_map_.prepareNextVersion(); for (const auto& removed_resource : removed_resources) { - ENVOY_LOG(trace, "Cilium removing network policy resource {}", removed_resource); + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); const auto* resource_entry = pending_resource_map.findEntry(removed_resource); if (resource_entry == nullptr) { continue; @@ -2731,7 +2731,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( continue; } throw EnvoyException( - fmt::format("Network Policy delta removed resource '{}' is a policy endpoint IP alias, " + fmt::format("NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " "not a resource name", removed_resource)); } @@ -2767,7 +2767,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( } ENVOY_LOG(debug, - "Received delta Network Policy selector resource {} in onConfigUpdate() " + "Received NetworkPolicyResource selector {} in onConfigUpdate() " "version {}", resource_name, system_version_info); auto selector_handle = @@ -2775,7 +2775,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( if (!pending_resource_map.emplace(resource_name, ResourceKey::selectorResource(selector_handle))) { throw EnvoyException(fmt::format( - "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " "stream: " "incoming selector resource '{}' collides with existing {}", system_version_info, resource_name, is_new_stream ? "a new" : "an old", resource_name, @@ -2794,13 +2794,13 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( case cilium::NetworkPolicyResource::kPolicy: { const auto& config = typed_resource.policy(); if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); + throw EnvoyException("NetworkPolicyResource has no endpoint ips"); } if (config.endpoint_id() == 0) { - throw EnvoyException("Network Policy endpoint_id must be non-zero"); + throw EnvoyException("NetworkPolicyResource endpoint_id must be non-zero"); } ENVOY_LOG(debug, - "Received delta Network Policy resource {} for endpoint {}, endpoint_ip {} in " + "Received NetworkPolicyResource {} for endpoint {}, endpoint_ip {} in " "onConfigUpdate() version {}", resource_name, config.endpoint_id(), config.endpoint_ips()[0], system_version_info); @@ -2809,7 +2809,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( &pending_resource_map); if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { throw EnvoyException(fmt::format( - "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " "stream: " "incoming {} collides with existing {}", system_version_info, resource_name, is_new_stream ? "a new" : "an old", @@ -2820,7 +2820,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", endpoint_ip); if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp())) { throw EnvoyException(fmt::format( - "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " "stream: " "incoming {} collides with existing {}", system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", @@ -2829,7 +2829,7 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( } if (!new_policy_map.emplace(endpoint_ip, policy).second) { throw EnvoyException(fmt::format( - "Network Policy delta update for version {} has duplicate resource key '{}' on {} " + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " "stream: " "incoming {} collides with existing {}", system_version_info, endpoint_ip, is_new_stream ? "a new" : "an old", @@ -2840,11 +2840,11 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( break; } case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: - throw EnvoyException("Network Policy delta resource has no payload"); + throw EnvoyException("NetworkPolicyResource has no payload"); } } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy delta update for version {} failed: {}", system_version_info, + ENVOY_LOG(warn, "NetworkPolicyResource update for version {} failed: {}", system_version_info, e.what()); stats_.updates_rejected_.inc(); removeInitManager(); @@ -2867,7 +2867,8 @@ void NetworkPolicyMapImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailu const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. - ENVOY_LOG(debug, "Network Policy Update failed, keeping existing policy."); + ENVOY_LOG(debug, "NetworkPolicy update on stream {} failed, keeping existing policy.", + streamGeneration()); } ProtobufTypes::MessagePtr diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index 0fe92abc4..c7f16a2db 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -445,7 +445,7 @@ TEST_F(CiliumNetworkPolicyTest, RejectsWhitespaceInSotwWrappedResourceName) { endpoint_id: 42 )EOF"), EnvoyException, - "Network Policy resource name 'policy 42' must not contain whitespace"); + "NetworkPolicy resource name 'policy 42' must not contain whitespace"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaIncrementalPolicyUpdate) { @@ -717,7 +717,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPolicyUpdateRejectsMissingSelectorReso - selectors: [ "selector-2" ] )EOF"), EnvoyException, - "Delta Network Policy rule references missing selector resource " + "NetworkPolicyResource rule references missing selector resource " "'selector-2'"); EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); @@ -870,7 +870,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBe remote_identities: [ 44 ] )EOF"), EnvoyException, - R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); @@ -1149,7 +1149,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsAddedResourceNamesWithWhitespac remote_identities: [ 43 ] )EOF"), EnvoyException, - "Network Policy delta resource name 'selector 1' must not contain whitespace"); + "NetworkPolicyResource added resource name 'selector 1' must not contain whitespace"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitespace) { @@ -1159,7 +1159,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitesp - "selector 1" )EOF"), EnvoyException, - "Network Policy delta removed resource name 'selector 1' must not contain whitespace"); + "NetworkPolicyResource removed resource name 'selector 1' must not contain whitespace"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { @@ -1184,7 +1184,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInS endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { @@ -1225,7 +1225,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdat endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { @@ -1250,7 +1250,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollision endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { @@ -1271,7 +1271,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesI remote_identities: [ 46, 47 ] )EOF"), EnvoyException, - R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { @@ -1359,7 +1359,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistin endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1389,7 +1389,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1419,7 +1419,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, endpoint_id: 43 )EOF"), EnvoyException, - R"(Network Policy delta update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1447,7 +1447,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, remote_identities: [ 43 ] )EOF"), EnvoyException, - R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, @@ -1475,7 +1475,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, remote_identities: [ 43 ] )EOF"), EnvoyException, - R"(Network Policy delta .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { @@ -1497,7 +1497,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) - "10.1.2.3" )EOF"), EnvoyException, - "Network Policy delta removed resource '10.1.2.3' is a policy endpoint IP alias, not a " + "NetworkPolicyResource removed resource '10.1.2.3' is a policy endpoint IP alias, not a " "resource name"); } @@ -1569,7 +1569,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsZeroEndpointIdRegardlessOfResou - "10.1.2.3" endpoint_id: 0 )EOF"), - EnvoyException, "Network Policy endpoint_id must be non-zero"); + EnvoyException, "NetworkPolicyResource endpoint_id must be non-zero"); } TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsInconsistentPassPrecedence) { From db12d3e26a86c4c47d072d86f411cf6760198b1e Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Sun, 19 Apr 2026 08:32:50 +0200 Subject: [PATCH 14/14] policy: Switch to delta mode when possible Switch to delta mode more eagerly when we have evidence that the agent is capable, but switch to SotW mode only when xDS stream transport had failed to connect or closes. Signed-off-by: Jarno Rajahalme --- cilium/network_policy.cc | 57 +++++++++---- tests/cilium_network_policy_test.cc | 119 +++++++++++++++++++++++++--- 2 files changed, 152 insertions(+), 24 deletions(-) diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 056fdfbd4..331318d95 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -253,8 +253,8 @@ class ResourceMapOverlay { } const auto* selector_entry = entry->selectorResourceEntry(); if (selector_entry == nullptr || selector_entry->handle == nullptr) { - throw EnvoyException( - fmt::format("NetworkPolicyResource rule references non-selector resource '{}'", selector)); + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references non-selector resource '{}'", selector)); } return selector_entry->handle; } @@ -407,7 +407,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, if (!subscription_connected_ && subscription_ != nullptr) { subscription_connected_ = grpcStreamConnected(subscription_.get()); } - maybeRecreateSubscriptionInDesiredMode(); + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); } protected: @@ -450,27 +450,57 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } void onSubscriptionTransportEstablished(uint64_t subscription_id) { - ++subscription_stream_generation_; - + // skip stale notifications for earlier subscriptions if (subscription_id != subscription_id_) { return; } + ++subscription_stream_generation_; + subscription_connected_ = true; } void onSubscriptionTransportClosed(uint64_t subscription_id) { + // skip stale notifications for earlier subscriptions if (subscription_id != subscription_id_) { return; } subscription_connected_ = false; - maybeRecreateSubscriptionInDesiredMode(); - } - void maybeRecreateSubscriptionInDesiredMode() { - if (subscription_ == nullptr || subscription_connected_ || - desired_use_delta_xds_ == subscription_use_delta_xds_) { + // Test code executes synchronously + if (subscription_factory_for_test_) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); return; } + + // The close callback runs on the subscription object's own stack, so defer any possible + // recreation until after it unwinds to avoid destroying the current subscription mid-callback. + context_.mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + // skip stale callbacks for earlier subscriptions + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + } + + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + // only ever skip subscribe if we have a subscription already, and it is already connected in + // delta or desired mode, or still connecting in desired mode. + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscription_use_delta_xds_) { + // Keep delta on a connected subscription until transport closes. + return; + } + if (subscription_use_delta_xds_ == desired_use_delta_xds_) { + // Let the current subscription keep going when it is already in the desired mode. + return; + } + } + + // Recreate the subscription in the latest desired mode. subscribe(); } @@ -2618,10 +2648,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource selector {}", resource); const auto* resource_entry = pending_resource_map.findEntry(resource); if (resource_entry == nullptr) { - ENVOY_LOG( - debug, - "NetworkPolicyResource removed selector name '{}' not found from resource map", - resource); + ENVOY_LOG(debug, + "NetworkPolicyResource removed selector name '{}' not found from resource map", + resource); continue; } if (resource_entry->isPolicyEndpointIpEntry()) { diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index c7f16a2db..4edcb758f 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -348,7 +348,7 @@ TEST_F(CiliumNetworkPolicyDeltaTest, ManagedSubscriptionColdStartUsesConfiguredD EXPECT_THAT(state->start_resources_.front(), testing::ElementsAre(std::string("*"))); } -TEST_F(CiliumNetworkPolicyTest, FlagFlipOnHealthySubscriptionWaitsForTransportClose) { +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromSotwToDeltaOnHealthySubscriptionRecreatesImmediately) { auto state = std::make_shared(); std::vector created_modes; setSubscriptionFactoryForTest( @@ -364,18 +364,57 @@ TEST_F(CiliumNetworkPolicyTest, FlagFlipOnHealthySubscriptionWaitsForTransportCl setUseDeltaXds(true); EXPECT_TRUE(configuredUseDeltaXds()); - EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); - EXPECT_TRUE(subscriptionConnectedForTest()); - EXPECT_THAT(created_modes, testing::ElementsAre(false)); - EXPECT_EQ(state->start_calls_, 1); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); +} - onSubscriptionTransportCloseForTest(); +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromDeltaToSotwOnHealthySubscriptionWaitsForClose) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); - EXPECT_FALSE(subscriptionConnectedForTest()); + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(true); + + EXPECT_TRUE(configuredUseDeltaXds()); EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); EXPECT_EQ(state->start_calls_, 2); EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_TRUE(subscriptionConnectedForTest()); + // Once we have an established delta subscription, keep it until transport close even if the + // configured desired mode flips back to SotW. + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); } TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { @@ -400,7 +439,66 @@ TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); } -TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipKeepsCurrentMode) { +TEST_F(CiliumNetworkPolicyDeltaTest, FlagFlipFromDisconnectedDeltaToSotwRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DowngradeFromConnectedDeltaRecreatesDisconnectedRetryToSotw) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + // Agent restart/downgrade drops the established delta transport. While the desired mode is still + // delta we recreate immediately and begin retrying delta. + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 2); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true)); + EXPECT_THAT(state->start_resources_.back(), testing::ElementsAre(std::string("*"))); + + // When listener metadata later reveals the downgraded agent no longer supports delta, flip to + // SotW immediately rather than letting the disconnected delta retry loop forever. + setUseDeltaXds(false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipRecreatesInCurrentMode) { auto state = std::make_shared(); std::vector created_modes; setSubscriptionFactoryForTest( @@ -416,8 +514,9 @@ TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipKeepsCurrentMode) { EXPECT_FALSE(subscriptionConnectedForTest()); EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); - EXPECT_THAT(created_modes, testing::ElementsAre(false)); - EXPECT_EQ(state->start_calls_, 1); + EXPECT_THAT(created_modes, testing::ElementsAre(false, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); } TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) {