diff --git a/manifests/golang.yml b/manifests/golang.yml index 1cade292075..3d1f435197a 100644 --- a/manifests/golang.yml +++ b/manifests/golang.yml @@ -1131,6 +1131,9 @@ manifest: tests/ffe/test_exposures.py: v2.6.0-dev # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0 tests/ffe/test_flag_eval_evp.py: v2.10.0-dev tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation::test_ffe_evp_flagevaluation_degradation: flaky (FFL-2676) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2784) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: missing_feature (FFL-2784) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: missing_feature (FFL-2784) tests/ffe/test_flag_eval_metrics.py: v2.8.0 tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Metric_Parse_Error_Invalid_Regex: irrelevant (Go validates regex at config load time) tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980) diff --git a/manifests/php.yml b/manifests/php.yml index 4a0f57c67c6..58bbae6ce9e 100644 --- a/manifests/php.yml +++ b/manifests/php.yml @@ -752,6 +752,9 @@ manifest: symfony7x: incomplete_test_app tests/ffe/test_exposures.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676) tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: bug (FFL-2676) + tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: bug (FFL-2676) tests/ffe/test_flag_eval_metrics.py: - weblog_declaration: "*": v1.21.0-dev diff --git a/tests/ffe/test_flag_eval_evp.py b/tests/ffe/test_flag_eval_evp.py index ff274b4c223..6f909598a6b 100644 --- a/tests/ffe/test_flag_eval_evp.py +++ b/tests/ffe/test_flag_eval_evp.py @@ -4,13 +4,13 @@ from concurrent.futures import ThreadPoolExecutor from typing import cast -import pytest from tests.ffe.utils.fixtures import JSON, make_ufc_fixture from utils import HttpResponse from utils import features from utils import interfaces from utils import remote_config as rc +from utils import scenario_crash from utils import scenarios from utils import weblog @@ -23,6 +23,18 @@ EVP_FULL_TIER_PER_FLAG_CAP = 10_000 EVP_DEGRADATION_OVERFLOW_EVALS = 2_000 +# Fixed input/output vector for the PII-protection tests. Every SDK's unit tests +# should assert against the same input to prove byte-identical hashing across SDKs. +PII_TARGETING_KEY = "jane.doe@datadoghq.com" +PII_TARGETING_KEY_HASHED = "sha256_b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b" +PII_ATTRIBUTES: dict[str, object] = { + "org_id": 1234, + "user_email": "jane.doe@datadoghq.com", + "plan": "enterprise", + "region": "us-east-1", + "account.tier": "gold", +} + def make_multi_flag_fixture(flag_keys: list[str]) -> JSON: fixture = make_ufc_fixture(flag_keys[0]) @@ -184,6 +196,33 @@ def event_identity(event: JSON) -> str: return json.dumps(visible_identity, sort_keys=True, default=str) +def _assert_hashed_targeting_key(event: JSON) -> None: + targeting_key = event.get("targeting_key") + assert targeting_key == PII_TARGETING_KEY_HASHED, ( + f"Expected hashed targeting_key {PII_TARGETING_KEY_HASHED}, got {targeting_key!r}" + ) + assert isinstance(targeting_key, str) + assert targeting_key.startswith("sha256_"), f"hashed targeting_key must start with 'sha256_': {targeting_key!r}" + assert len(targeting_key) == 71, f"hashed targeting_key must be 71 chars, got {len(targeting_key)}" + hex_suffix = targeting_key[len("sha256_") :] + assert len(hex_suffix) == 64, f"hashed targeting_key suffix must be 64 chars: {targeting_key!r}" + assert all(c in "0123456789abcdef" for c in hex_suffix), ( + f"hashed targeting_key suffix must be lowercase hex: {targeting_key!r}" + ) + + +def assert_no_raw_pii_in_event(event: JSON, forbidden_values: list[str]) -> None: + """Walk the entire serialized event and assert none of the raw PII strings appear anywhere. + + Guards against SDK bugs that route unhashed values into unexpected fields (e.g., a raw + email leaking into ``context.user_email`` even when ``context.evaluation`` is correctly + omitted). + """ + serialized = json.dumps(event, default=str) + for value in forbidden_values: + assert value not in serialized, f"raw PII value {value!r} must not appear anywhere in event: {event}" + + def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: seen_by_batch: dict[int, set[str]] = {} duplicates: set[str] = set() @@ -199,7 +238,6 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail class Test_FFE_EVP_Flagevaluation_Basic: """Test that flag evaluation produces an EVP flagevaluation payload.""" @@ -226,7 +264,6 @@ def test_ffe_evp_flagevaluation_basic(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail class Test_FFE_EVP_Flagevaluation_Count: """Test that repeated evaluations are counted in EVP flagevaluation payloads.""" @@ -257,7 +294,6 @@ def test_ffe_evp_flagevaluation_count(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail class Test_FFE_EVP_Flagevaluation_Context_Bounds: """Test that EVP evaluation context is bounded before it reaches payloads.""" @@ -303,7 +339,6 @@ def test_ffe_evp_flagevaluation_context_bounds(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail class Test_FFE_EVP_Flagevaluation_Runtime_Default: """Test that runtime defaults are surfaced without OpenFeature reason.""" @@ -330,7 +365,6 @@ def test_ffe_evp_flagevaluation_runtime_default(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail class Test_FFE_EVP_Flagevaluation_Load_Aggregation: """Test CI-safe load aggregation without treating system-tests as a perf test.""" @@ -371,7 +405,7 @@ def test_ffe_evp_flagevaluation_load_aggregation(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail +@scenario_crash class Test_FFE_EVP_Flagevaluation_Burst_Aggregation: """Test a bounded request burst through the async EVP aggregation path.""" @@ -410,7 +444,7 @@ def test_ffe_evp_flagevaluation_burst_aggregation(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail +@scenario_crash class Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: """Test many full-tier aggregation buckets stay distinct and counted.""" @@ -450,7 +484,7 @@ def test_ffe_evp_flagevaluation_high_cardinality_aggregation(self) -> None: @scenarios.feature_flagging_and_experimentation @features.feature_flags_evp_flagevaluation -@pytest.mark.skip_if_xfail +@scenario_crash class Test_FFE_EVP_Flagevaluation_Degradation: """Test degraded EVP shape after the production per-flag full-tier cap is exceeded.""" @@ -489,3 +523,139 @@ def test_ffe_evp_flagevaluation_degradation(self) -> None: assert "targeting_key" not in event, f"degraded event must omit targeting_key: {event}" assert object_key(event.get("variant"), "variant") == "on" assert object_key(event.get("allocation"), "allocation") == "default-allocation" + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_evp_flagevaluation +class Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: + """Test that when observeFullEvaluationData is absent from UFC, targeting_key is hashed and context.evaluation is omitted (default PII-protection).""" + + def setup_ffe_evp_flagevaluation_observe_full_data_absent(self) -> None: + config_id = "ffe-evp-observe-absent" + self.flag_key = "evp-observe-absent-flag" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", + make_ufc_fixture(self.flag_key), + ).apply() + + self.r = evaluate_flag( + self.flag_key, + targeting_key=PII_TARGETING_KEY, + attributes=PII_ATTRIBUTES, + ) + + def test_ffe_evp_flagevaluation_observe_full_data_absent(self) -> None: + assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" + + wait_for_evp_flagevaluation_event(self.flag_key) + events = find_evp_flagevaluation_events(self.flag_key) + assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}" + + forbidden_raw_values = [PII_TARGETING_KEY, *[str(v) for v in PII_ATTRIBUTES.values()]] + + for batch, event in events: + assert_batch_context(batch) + assert_event_contract(event, self.flag_key) + _assert_hashed_targeting_key(event) + + context = event.get("context") + if isinstance(context, dict): + assert "evaluation" not in context, ( + f"context.evaluation must be omitted when observeFullEvaluationData is absent: {context}" + ) + + assert_no_raw_pii_in_event(event, forbidden_raw_values) + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_evp_flagevaluation +class Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: + """Test that when observeFullEvaluationData=false in UFC, targeting_key is hashed and context.evaluation is omitted.""" + + def setup_ffe_evp_flagevaluation_observe_full_data_false(self) -> None: + config_id = "ffe-evp-observe-false" + self.flag_key = "evp-observe-false-flag" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", + make_ufc_fixture(self.flag_key, observe_full_evaluation_data=False), + ).apply() + + self.r = evaluate_flag( + self.flag_key, + targeting_key=PII_TARGETING_KEY, + attributes=PII_ATTRIBUTES, + ) + + def test_ffe_evp_flagevaluation_observe_full_data_false(self) -> None: + assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" + + wait_for_evp_flagevaluation_event(self.flag_key) + events = find_evp_flagevaluation_events(self.flag_key) + assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}" + + forbidden_raw_values = [PII_TARGETING_KEY, *[str(v) for v in PII_ATTRIBUTES.values()]] + + for batch, event in events: + assert_batch_context(batch) + assert_event_contract(event, self.flag_key) + _assert_hashed_targeting_key(event) + + context = event.get("context") + if isinstance(context, dict): + assert "evaluation" not in context, ( + f"context.evaluation must be omitted when observeFullEvaluationData=false: {context}" + ) + + assert_no_raw_pii_in_event(event, forbidden_raw_values) + + +@scenarios.feature_flagging_and_experimentation +@features.feature_flags_evp_flagevaluation +class Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: + """Test that when observeFullEvaluationData=true in UFC, targeting_key is raw and context.evaluation is populated.""" + + def setup_ffe_evp_flagevaluation_observe_full_data_true(self) -> None: + config_id = "ffe-evp-observe-true" + self.flag_key = "evp-observe-true-flag" + rc.tracer_rc_state.reset().set_config( + f"{RC_PATH}/{config_id}/config", + make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True), + ).apply() + + self.r = evaluate_flag( + self.flag_key, + targeting_key=PII_TARGETING_KEY, + attributes=PII_ATTRIBUTES, + ) + + def test_ffe_evp_flagevaluation_observe_full_data_true(self) -> None: + assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}" + + wait_for_evp_flagevaluation_event(self.flag_key) + events = find_evp_flagevaluation_events(self.flag_key) + assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}" + + for batch, event in events: + assert_batch_context(batch) + assert_event_contract(event, self.flag_key) + + targeting_key = event.get("targeting_key") + assert targeting_key == PII_TARGETING_KEY, ( + f"Expected raw targeting_key {PII_TARGETING_KEY!r}, got {targeting_key!r}" + ) + assert isinstance(targeting_key, str) + assert not targeting_key.startswith("sha256_"), ( + f"unhashed targeting_key must not start with 'sha256_': {targeting_key!r}" + ) + + context = event.get("context") + assert isinstance(context, dict), f"context must be an object when observeFullEvaluationData=true: {event}" + evaluation_context = context.get("evaluation") + assert isinstance(evaluation_context, dict), ( + f"context.evaluation must be an object when observeFullEvaluationData=true: {context}" + ) + for key, expected_value in PII_ATTRIBUTES.items(): + assert key in evaluation_context, f"context.evaluation missing attribute {key!r}: {evaluation_context}" + assert evaluation_context[key] == expected_value, ( + f"context.evaluation[{key!r}] expected {expected_value!r}, got {evaluation_context[key]!r}" + ) diff --git a/tests/ffe/utils/fixtures.py b/tests/ffe/utils/fixtures.py index 6342544a990..2813a34eb0c 100644 --- a/tests/ffe/utils/fixtures.py +++ b/tests/ffe/utils/fixtures.py @@ -23,10 +23,11 @@ def make_ufc_fixture( enabled: bool = True, allocation_key: str = "default-allocation", variation_values: dict[str, VariationValue] | None = None, + observe_full_evaluation_data: bool | None = None, ) -> JSON: values = variation_values or DEFAULT_VARIATION_VALUES[variation_type] - return { + ufc: JSON = { "createdAt": "2024-04-17T19:40:53.716Z", "format": "SERVER", "environment": {"name": "Test"}, @@ -47,6 +48,9 @@ def make_ufc_fixture( } }, } + if observe_full_evaluation_data is not None: + ufc["observeFullEvaluationData"] = observe_full_evaluation_data + return ufc def make_exposure_ufc_fixture(