FFL-2783: add tests for observeFullEvaluationData PII protection (gated as missing_feature)

manifests/golang.yml

@@ -1131,6 +1131,9 @@ manifest:
   tests/ffe/test_exposures.py: v2.6.0-dev  # Easy win for chi, echo, gin, net-http, net-http-orchestrion, uds-echo and version 2.5.0
   tests/ffe/test_flag_eval_evp.py: v2.10.0-dev
   tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation::test_ffe_evp_flagevaluation_degradation: flaky (FFL-2676)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed: missing_feature (FFL-2784)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed: missing_feature (FFL-2784)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed: missing_feature (FFL-2784)
   tests/ffe/test_flag_eval_metrics.py: v2.8.0
   tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Metric_Parse_Error_Invalid_Regex: irrelevant (Go validates regex at config load time)
   tests/ffe/test_flag_eval_metrics.py::Test_FFE_Eval_Nested_Attributes_Ignored: irrelevant (FFL-1980)

manifests/php.yml

@@ -752,6 +752,9 @@ manifest:
         symfony7x: incomplete_test_app
   tests/ffe/test_exposures.py::Test_FFE_Exposure_Events::test_ffe_multiple_remote_config_files: flaky (FFL-2676)
   tests/ffe/test_flag_eval_evp.py: missing_feature (FFL-2446)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Burst_Aggregation: bug (FFL-2676)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_Degradation: bug (FFL-2676)
+  tests/ffe/test_flag_eval_evp.py::Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation: bug (FFL-2676)
   tests/ffe/test_flag_eval_metrics.py:
     - weblog_declaration:
         "*": v1.21.0-dev

tests/ffe/test_flag_eval_evp.py

@@ -4,13 +4,13 @@
 from concurrent.futures import ThreadPoolExecutor
 from typing import cast
 
-import pytest
 
 from tests.ffe.utils.fixtures import JSON, make_ufc_fixture
 from utils import HttpResponse
 from utils import features
 from utils import interfaces
 from utils import remote_config as rc
+from utils import scenario_crash
 from utils import scenarios
 from utils import weblog
 
@@ -23,6 +23,18 @@
 EVP_FULL_TIER_PER_FLAG_CAP = 10_000
 EVP_DEGRADATION_OVERFLOW_EVALS = 2_000
 
+# Fixed input/output vector for the PII-protection tests. Every SDK's unit tests
+# should assert against the same input to prove byte-identical hashing across SDKs.
+PII_TARGETING_KEY = "jane.doe@datadoghq.com"
+PII_TARGETING_KEY_HASHED = "sha256_b4698f9b6d186781fa8dc59e533578fa2d8379a46b1cf6db85cda6aa9c99e51b"
+PII_ATTRIBUTES: dict[str, object] = {
+    "org_id": 1234,
+    "user_email": "jane.doe@datadoghq.com",
+    "plan": "enterprise",
+    "region": "us-east-1",
+    "account.tier": "gold",
+}
+
 
 def make_multi_flag_fixture(flag_keys: list[str]) -> JSON:
     fixture = make_ufc_fixture(flag_keys[0])
@@ -184,6 +196,33 @@ def event_identity(event: JSON) -> str:
     return json.dumps(visible_identity, sort_keys=True, default=str)
 
 
+def _assert_hashed_targeting_key(event: JSON) -> None:
+    targeting_key = event.get("targeting_key")
+    assert targeting_key == PII_TARGETING_KEY_HASHED, (
+        f"Expected hashed targeting_key {PII_TARGETING_KEY_HASHED}, got {targeting_key!r}"
+    )
+    assert isinstance(targeting_key, str)
+    assert targeting_key.startswith("sha256_"), f"hashed targeting_key must start with 'sha256_': {targeting_key!r}"
+    assert len(targeting_key) == 71, f"hashed targeting_key must be 71 chars, got {len(targeting_key)}"
+    hex_suffix = targeting_key[len("sha256_") :]
+    assert len(hex_suffix) == 64, f"hashed targeting_key suffix must be 64 chars: {targeting_key!r}"
+    assert all(c in "0123456789abcdef" for c in hex_suffix), (
+        f"hashed targeting_key suffix must be lowercase hex: {targeting_key!r}"
+    )
+
+
+def assert_no_raw_pii_in_event(event: JSON, forbidden_values: list[str]) -> None:
+    """Walk the entire serialized event and assert none of the raw PII strings appear anywhere.
+
+    Guards against SDK bugs that route unhashed values into unexpected fields (e.g., a raw
+    email leaking into ``context.user_email`` even when ``context.evaluation`` is correctly
+    omitted).
+    """
+    serialized = json.dumps(event, default=str)
+    for value in forbidden_values:
+        assert value not in serialized, f"raw PII value {value!r} must not appear anywhere in event: {event}"
+
+
 def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None:
     seen_by_batch: dict[int, set[str]] = {}
     duplicates: set[str] = set()
@@ -199,7 +238,6 @@ def assert_no_duplicate_visible_events(events: list[tuple[JSON, JSON]]) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
 class Test_FFE_EVP_Flagevaluation_Basic:
     """Test that flag evaluation produces an EVP flagevaluation payload."""
 
@@ -226,7 +264,6 @@ def test_ffe_evp_flagevaluation_basic(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
 class Test_FFE_EVP_Flagevaluation_Count:
     """Test that repeated evaluations are counted in EVP flagevaluation payloads."""
 
@@ -257,7 +294,6 @@ def test_ffe_evp_flagevaluation_count(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
 class Test_FFE_EVP_Flagevaluation_Context_Bounds:
     """Test that EVP evaluation context is bounded before it reaches payloads."""
 
@@ -303,7 +339,6 @@ def test_ffe_evp_flagevaluation_context_bounds(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
 class Test_FFE_EVP_Flagevaluation_Runtime_Default:
     """Test that runtime defaults are surfaced without OpenFeature reason."""
 
@@ -330,7 +365,6 @@ def test_ffe_evp_flagevaluation_runtime_default(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
 class Test_FFE_EVP_Flagevaluation_Load_Aggregation:
     """Test CI-safe load aggregation without treating system-tests as a perf test."""
 
@@ -371,7 +405,7 @@ def test_ffe_evp_flagevaluation_load_aggregation(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
+@scenario_crash
 class Test_FFE_EVP_Flagevaluation_Burst_Aggregation:
     """Test a bounded request burst through the async EVP aggregation path."""
 
@@ -410,7 +444,7 @@ def test_ffe_evp_flagevaluation_burst_aggregation(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
+@scenario_crash
 class Test_FFE_EVP_Flagevaluation_High_Cardinality_Aggregation:
     """Test many full-tier aggregation buckets stay distinct and counted."""
 
@@ -450,7 +484,7 @@ def test_ffe_evp_flagevaluation_high_cardinality_aggregation(self) -> None:
 
 @scenarios.feature_flagging_and_experimentation
 @features.feature_flags_evp_flagevaluation
-@pytest.mark.skip_if_xfail
+@scenario_crash
 class Test_FFE_EVP_Flagevaluation_Degradation:
     """Test degraded EVP shape after the production per-flag full-tier cap is exceeded."""
 
@@ -489,3 +523,139 @@ def test_ffe_evp_flagevaluation_degradation(self) -> None:
             assert "targeting_key" not in event, f"degraded event must omit targeting_key: {event}"
             assert object_key(event.get("variant"), "variant") == "on"
             assert object_key(event.get("allocation"), "allocation") == "default-allocation"
+
+
+@scenarios.feature_flagging_and_experimentation
+@features.feature_flags_evp_flagevaluation
+class Test_FFE_EVP_Flagevaluation_ObserveFullData_Absent_Hashed:
+    """Test that when observeFullEvaluationData is absent from UFC, targeting_key is hashed and context.evaluation is omitted (default PII-protection)."""
+
+    def setup_ffe_evp_flagevaluation_observe_full_data_absent(self) -> None:
+        config_id = "ffe-evp-observe-absent"
+        self.flag_key = "evp-observe-absent-flag"
+        rc.tracer_rc_state.reset().set_config(
+            f"{RC_PATH}/{config_id}/config",
+            make_ufc_fixture(self.flag_key),
+        ).apply()
+
+        self.r = evaluate_flag(
+            self.flag_key,
+            targeting_key=PII_TARGETING_KEY,
+            attributes=PII_ATTRIBUTES,
+        )
+
+    def test_ffe_evp_flagevaluation_observe_full_data_absent(self) -> None:
+        assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}"
+
+        wait_for_evp_flagevaluation_event(self.flag_key)
+        events = find_evp_flagevaluation_events(self.flag_key)
+        assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}"
+
+        forbidden_raw_values = [PII_TARGETING_KEY, *[str(v) for v in PII_ATTRIBUTES.values()]]
+
+        for batch, event in events:
+            assert_batch_context(batch)
+            assert_event_contract(event, self.flag_key)
+            _assert_hashed_targeting_key(event)
+
+            context = event.get("context")
+            if isinstance(context, dict):
+                assert "evaluation" not in context, (
+                    f"context.evaluation must be omitted when observeFullEvaluationData is absent: {context}"
+                )
+
+            assert_no_raw_pii_in_event(event, forbidden_raw_values)
+
+
+@scenarios.feature_flagging_and_experimentation
+@features.feature_flags_evp_flagevaluation
+class Test_FFE_EVP_Flagevaluation_ObserveFullData_False_Hashed:
+    """Test that when observeFullEvaluationData=false in UFC, targeting_key is hashed and context.evaluation is omitted."""
+
+    def setup_ffe_evp_flagevaluation_observe_full_data_false(self) -> None:
+        config_id = "ffe-evp-observe-false"
+        self.flag_key = "evp-observe-false-flag"
+        rc.tracer_rc_state.reset().set_config(
+            f"{RC_PATH}/{config_id}/config",
+            make_ufc_fixture(self.flag_key, observe_full_evaluation_data=False),
+        ).apply()
+
+        self.r = evaluate_flag(
+            self.flag_key,
+            targeting_key=PII_TARGETING_KEY,
+            attributes=PII_ATTRIBUTES,
+        )
+
+    def test_ffe_evp_flagevaluation_observe_full_data_false(self) -> None:
+        assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}"
+
+        wait_for_evp_flagevaluation_event(self.flag_key)
+        events = find_evp_flagevaluation_events(self.flag_key)
+        assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}"
+
+        forbidden_raw_values = [PII_TARGETING_KEY, *[str(v) for v in PII_ATTRIBUTES.values()]]
+
+        for batch, event in events:
+            assert_batch_context(batch)
+            assert_event_contract(event, self.flag_key)
+            _assert_hashed_targeting_key(event)
+
+            context = event.get("context")
+            if isinstance(context, dict):
+                assert "evaluation" not in context, (
+                    f"context.evaluation must be omitted when observeFullEvaluationData=false: {context}"
+                )
+
+            assert_no_raw_pii_in_event(event, forbidden_raw_values)
+
+
+@scenarios.feature_flagging_and_experimentation
+@features.feature_flags_evp_flagevaluation
+class Test_FFE_EVP_Flagevaluation_ObserveFullData_True_Unhashed:
+    """Test that when observeFullEvaluationData=true in UFC, targeting_key is raw and context.evaluation is populated."""
+
+    def setup_ffe_evp_flagevaluation_observe_full_data_true(self) -> None:
+        config_id = "ffe-evp-observe-true"
+        self.flag_key = "evp-observe-true-flag"
+        rc.tracer_rc_state.reset().set_config(
+            f"{RC_PATH}/{config_id}/config",
+            make_ufc_fixture(self.flag_key, observe_full_evaluation_data=True),
+        ).apply()
+
+        self.r = evaluate_flag(
+            self.flag_key,
+            targeting_key=PII_TARGETING_KEY,
+            attributes=PII_ATTRIBUTES,
+        )
+
+    def test_ffe_evp_flagevaluation_observe_full_data_true(self) -> None:
+        assert self.r.status_code == 200, f"Flag evaluation failed: {self.r.text}"
+
+        wait_for_evp_flagevaluation_event(self.flag_key)
+        events = find_evp_flagevaluation_events(self.flag_key)
+        assert events, f"Expected EVP flagevaluation event for flag {self.flag_key}"
+
+        for batch, event in events:
+            assert_batch_context(batch)
+            assert_event_contract(event, self.flag_key)
+
+            targeting_key = event.get("targeting_key")
+            assert targeting_key == PII_TARGETING_KEY, (
+                f"Expected raw targeting_key {PII_TARGETING_KEY!r}, got {targeting_key!r}"
+            )
+            assert isinstance(targeting_key, str)
+            assert not targeting_key.startswith("sha256_"), (
+                f"unhashed targeting_key must not start with 'sha256_': {targeting_key!r}"
+            )
+
+            context = event.get("context")
+            assert isinstance(context, dict), f"context must be an object when observeFullEvaluationData=true: {event}"
+            evaluation_context = context.get("evaluation")
+            assert isinstance(evaluation_context, dict), (
+                f"context.evaluation must be an object when observeFullEvaluationData=true: {context}"
+            )
+            for key, expected_value in PII_ATTRIBUTES.items():
+                assert key in evaluation_context, f"context.evaluation missing attribute {key!r}: {evaluation_context}"
+                assert evaluation_context[key] == expected_value, (
+                    f"context.evaluation[{key!r}] expected {expected_value!r}, got {evaluation_context[key]!r}"
+                )

tests/ffe/utils/fixtures.py

@@ -23,10 +23,11 @@ def make_ufc_fixture(
     enabled: bool = True,
     allocation_key: str = "default-allocation",
     variation_values: dict[str, VariationValue] | None = None,
+    observe_full_evaluation_data: bool | None = None,
 ) -> JSON:
     values = variation_values or DEFAULT_VARIATION_VALUES[variation_type]
 
-    return {
+    ufc: JSON = {
         "createdAt": "2024-04-17T19:40:53.716Z",
         "format": "SERVER",
         "environment": {"name": "Test"},
@@ -47,6 +48,9 @@ def make_ufc_fixture(
             }
         },
     }
+    if observe_full_evaluation_data is not None:
+        ufc["observeFullEvaluationData"] = observe_full_evaluation_data
+    return ufc
 
 
 def make_exposure_ufc_fixture(