From 1bab8ed8d525cd749b6c69903557b4b3c43fab2a Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Tue, 26 May 2026 23:00:23 -0400 Subject: [PATCH 1/5] Add FFE first remote config request test --- tests/ffe/test_dynamic_evaluation.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/ffe/test_dynamic_evaluation.py b/tests/ffe/test_dynamic_evaluation.py index a3d9e671df8..2e63ac8caad 100644 --- a/tests/ffe/test_dynamic_evaluation.py +++ b/tests/ffe/test_dynamic_evaluation.py @@ -1,10 +1,12 @@ """Test feature flags dynamic evaluation via Remote Config.""" +import base64 import copy import json import uuid from http import HTTPStatus +from utils.dd_constants import Capabilities from utils import ( weblog, interfaces, @@ -19,6 +21,11 @@ RC_PATH = f"datadog/2/{RC_PRODUCT}" +def _decode_capabilities(capabilities: list[int] | str) -> int: + raw_capabilities = bytes(capabilities) if isinstance(capabilities, list) else base64.b64decode(capabilities) + return int.from_bytes(raw_capabilities, byteorder="big") + + # Simple UFC fixture for testing with doLog: true UFC_FIXTURE_DATA = { "createdAt": "2024-04-17T19:40:53.716Z", @@ -43,6 +50,37 @@ } +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_dynamic_evaluation +class Test_FFE_First_Remote_Config_Request: + """The tracer must subscribe to FFE on its first RC request. + + This covers the warm-Agent case: the Agent can already be running without + any FFE cache, then a tracer starts. If the tracer only adds FFE on a later + RC poll, the Agent can miss the fast new-client backend fetch and the app + may keep default flag values during startup. + """ + + def test_first_remote_config_request_subscribes_to_ffe(self) -> None: + remote_config_requests = list(interfaces.library.get_data(path_filters="/v0.7/config")) + assert remote_config_requests, "Expected the tracer to send at least one remote config request" + + first_request = remote_config_requests[0] + client = first_request["request"]["content"]["client"] + products = client.get("products", []) + assert RC_PRODUCT in products, ( + f"Expected first remote config request to subscribe to {RC_PRODUCT}, got products={products}. " + "If FFE appears only on a later poll, an already-running Agent can miss its new-client backend fetch." + ) + + capabilities = client.get("capabilities") + assert capabilities is not None, "Expected first remote config request to include capabilities" + decoded_capabilities = _decode_capabilities(capabilities) + assert (decoded_capabilities >> Capabilities.FFE_FLAG_CONFIGURATION_RULES) & 1 == 1, ( + f"Expected first remote config request to advertise {Capabilities.FFE_FLAG_CONFIGURATION_RULES.name}." + ) + + @scenarios.feature_flagging_and_experimentation @features.feature_flags_dynamic_evaluation class Test_FFE_Unknown_Operator_Tolerance: From 59b8c09d16599ae14b0115fe59c89a51ccd14898 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 27 May 2026 10:52:34 -0400 Subject: [PATCH 2/5] Add FFE remote config recovery test --- manifests/java.yml | 2 + manifests/python.yml | 2 + tests/ffe/test_dynamic_evaluation.py | 76 ++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/manifests/java.yml b/manifests/java.yml index 2f1f7bdc047..65eea0576e4 100644 --- a/manifests/java.yml +++ b/manifests/java.yml @@ -3154,7 +3154,9 @@ manifest: - weblog_declaration: "*": irrelevant spring-boot: v1.56.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: v1.63.0-SNAPSHOT tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation::test_valid_flag_unaffected: bug (FFL-2184) + tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_Then_Up: v1.63.0-SNAPSHOT tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) tests/ffe/test_exposures.py: - weblog_declaration: diff --git a/manifests/python.yml b/manifests/python.yml index c320776ad7f..60a635f89ba 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1274,6 +1274,8 @@ manifest: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 + tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_Then_Up: v4.11.0-dev + tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: v4.11.0-dev tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev tests/ffe/test_flag_eval_metrics.py: v4.8.0-dev diff --git a/tests/ffe/test_dynamic_evaluation.py b/tests/ffe/test_dynamic_evaluation.py index 2e63ac8caad..ee788b1a243 100644 --- a/tests/ffe/test_dynamic_evaluation.py +++ b/tests/ffe/test_dynamic_evaluation.py @@ -81,6 +81,82 @@ def test_first_remote_config_request_subscribes_to_ffe(self) -> None: ) +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_dynamic_evaluation +class Test_FFE_RC_Down_Then_Up: + """FFE must recover when RC is unavailable before the tracer receives flags. + + This covers the startup race we care about: the app starts, the tracer cannot + get FFE config from the Agent/RC endpoint yet, and evaluations correctly use + defaults. Once RC becomes available and sends FFE_FLAGS, the same app must + start using the delivered flag value without a restart. + """ + + def setup_ffe_rc_down_then_up_recovers(self): + self.config_request_data = None + self.flag_key = "test-flag" + self.default_before_config = "default-before-config" + self.default_after_config = "default-after-config" + + def wait_for_config_503(data: dict) -> bool: + if data["path"] == "/v0.7/config" and data["response"]["status_code"] == HTTPStatus.SERVICE_UNAVAILABLE: + self.config_request_data = data + return True + return False + + StaticJsonMockedTracerResponse( + path="/v0.7/config", mocked_json={"error": "Service Unavailable"}, status_code=503 + ).send() + + interfaces.library.wait_for(wait_for_config_503, timeout=60) + + self.default_eval = weblog.post( + "/ffe", + json={ + "flag": self.flag_key, + "variationType": "STRING", + "defaultValue": self.default_before_config, + "targetingKey": "user-before-rc-recovery", + "attributes": {}, + }, + ) + + self.config_state = ( + rc.tracer_rc_state.reset() + .set_config(f"{RC_PATH}/ffe-rc-down-then-up/config", UFC_FIXTURE_DATA) + .apply() + ) + + self.recovered_eval = weblog.post( + "/ffe", + json={ + "flag": self.flag_key, + "variationType": "STRING", + "defaultValue": self.default_after_config, + "targetingKey": "user-after-rc-recovery", + "attributes": {}, + }, + ) + + def test_ffe_rc_down_then_up_recovers(self): + assert self.config_request_data is not None, "No /v0.7/config 503 response was captured" + assert self.config_request_data["response"]["status_code"] == HTTPStatus.SERVICE_UNAVAILABLE, ( + f"Expected 503, got {self.config_request_data['response']['status_code']}" + ) + + assert self.default_eval.status_code == 200, f"Default evaluation failed: {self.default_eval.text}" + default_result = json.loads(self.default_eval.text) + assert default_result["value"] == self.default_before_config, ( + f"Expected default before config recovery, got '{default_result['value']}'" + ) + + assert self.recovered_eval.status_code == 200, f"Recovered evaluation failed: {self.recovered_eval.text}" + recovered_result = json.loads(self.recovered_eval.text) + assert recovered_result["value"] == "on", ( + f"Expected delivered flag value after config recovery, got '{recovered_result['value']}'" + ) + + @scenarios.feature_flagging_and_experimentation @features.feature_flags_dynamic_evaluation class Test_FFE_Unknown_Operator_Tolerance: From 155bd69c9c01ac92f303ace921bfdd1c002ff563 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 27 May 2026 11:13:16 -0400 Subject: [PATCH 3/5] Format FFE recovery system test --- tests/ffe/test_dynamic_evaluation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/ffe/test_dynamic_evaluation.py b/tests/ffe/test_dynamic_evaluation.py index ee788b1a243..0099ce798d3 100644 --- a/tests/ffe/test_dynamic_evaluation.py +++ b/tests/ffe/test_dynamic_evaluation.py @@ -122,9 +122,7 @@ def wait_for_config_503(data: dict) -> bool: ) self.config_state = ( - rc.tracer_rc_state.reset() - .set_config(f"{RC_PATH}/ffe-rc-down-then-up/config", UFC_FIXTURE_DATA) - .apply() + rc.tracer_rc_state.reset().set_config(f"{RC_PATH}/ffe-rc-down-then-up/config", UFC_FIXTURE_DATA).apply() ) self.recovered_eval = weblog.post( From 257aadd19200b7b3a231b36cc8b7d0eb8ad3ef01 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 27 May 2026 11:20:17 -0400 Subject: [PATCH 4/5] Sort FFE Python manifest entries --- manifests/python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/python.yml b/manifests/python.yml index 60a635f89ba..7ae0f250e28 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1273,9 +1273,9 @@ manifest: tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' + tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: v4.11.0-dev tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_Then_Up: v4.11.0-dev - tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: v4.11.0-dev tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622) tests/ffe/test_exposures.py: v4.2.0-dev tests/ffe/test_flag_eval_metrics.py: v4.8.0-dev From 022c28a1e95da4fbe887ce52d5a6e9a0df740531 Mon Sep 17 00:00:00 2001 From: Leo Romanovsky Date: Wed, 27 May 2026 11:36:54 -0400 Subject: [PATCH 5/5] Mark FFE first RC gaps in manifests --- manifests/dotnet.yml | 1 + manifests/python.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml index 4a2f1479cfe..79f673f7846 100644 --- a/manifests/dotnet.yml +++ b/manifests/dotnet.yml @@ -700,6 +700,7 @@ manifest: tests/docker_ssi/test_docker_ssi_appsec.py::TestDockerSSIAppsecFeatures::test_telemetry_source_ssi: v3.36.0 tests/ffe/test_dynamic_evaluation.py: v3.36.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_Flag_Parse_Error_Isolation: bug (FFL-2184) + tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_Then_Up: bug (FFL-2339) tests/ffe/test_dynamic_evaluation.py::Test_FFE_Unknown_Operator_Tolerance: bug (FFL-2184) tests/ffe/test_exposures.py: v3.36.0 tests/ffe/test_flag_eval_metrics.py: missing_feature (FFL-2257 - Requires Datadog.FeatureFlags.OpenFeature 2.3.0 NuGet package to be published) diff --git a/manifests/python.yml b/manifests/python.yml index 7ae0f250e28..886da9b90be 100644 --- a/manifests/python.yml +++ b/manifests/python.yml @@ -1273,7 +1273,7 @@ manifest: tests/docker_ssi/test_docker_ssi_crash.py::TestDockerSSICrash::test_crash: - declaration: bug (INPLAT-603) component_version: '>=3.0.0-dev' - tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: v4.11.0-dev + tests/ffe/test_dynamic_evaluation.py::Test_FFE_First_Remote_Config_Request: bug (FFL-2339) tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_From_Start: v4.0.0 tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Down_Then_Up: v4.11.0-dev tests/ffe/test_dynamic_evaluation.py::Test_FFE_RC_Unavailable: flaky (FFL-1622)