diff --git a/tests/ai_guard/test_ai_guard_sdk.py b/tests/ai_guard/test_ai_guard_sdk.py index ebeca4a450a..59634bc8941 100644 --- a/tests/ai_guard/test_ai_guard_sdk.py +++ b/tests/ai_guard/test_ai_guard_sdk.py @@ -2,7 +2,7 @@ from utils import context, interfaces, scenarios, weblog, features from utils.dd_constants import SamplingPriority -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan BLOCKING_HEADER: str = "X-AI-Guard-Block" MESSAGES: dict = { @@ -60,7 +60,7 @@ def _assert_key(values: dict, key: str, value: object | None = None): @scenarios.ai_guard class Test_Evaluation: def _assert_span(self, action: str, messages: list, *, blocking: str): - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span["resource"] != "ai_guard": return False @@ -201,7 +201,7 @@ def test_root_span_user_keep(self): @scenarios.ai_guard class Test_Full_Response_And_Tags: def _assert_span(self, response: dict, action: str): - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span["resource"] != "ai_guard": return False @@ -242,7 +242,7 @@ def test_evaluation(self): @features.ai_guard @scenarios.default class Test_SDK_Disabled: - def _validate_no_ai_guard_span(self, span: DataDogSpan): + def _validate_no_ai_guard_span(self, span: DataDogLibrarySpan): assert span["resource"] != "ai_guard" return True @@ -268,7 +268,7 @@ class Test_ContentParts: """Test AI Guard with multi-modal content parts (text + image_url).""" def _assert_span_with_content_parts(self, messages: list): - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span["resource"] != "ai_guard": return False diff --git a/tests/apm_tracing_e2e/test_otel.py b/tests/apm_tracing_e2e/test_otel.py index 82747623151..1910be0af59 100644 --- a/tests/apm_tracing_e2e/test_otel.py +++ b/tests/apm_tracing_e2e/test_otel.py @@ -1,5 +1,5 @@ from utils import weblog, scenarios, interfaces, features -from utils.dd_constants import TraceAgentPayloadFormat +from utils.dd_types import DataDogAgentSpan @features.otel_api @@ -26,20 +26,20 @@ def test_datadog_otel_span(self): assert len(spans) >= 2, "Agent did not submit the spans we want!" # Assert the parent span sent by the agent. - parent, parent_span_format = _get_span_by_resource(spans, "root-otel-name.dd-resource") + parent = _get_span_by_resource(spans, "root-otel-name.dd-resource") assert parent.get("parentID") is None - parent_meta = interfaces.agent.get_span_meta(parent, parent_span_format) + parent_meta = parent.meta if parent_meta["language"] != "jvm": # Java OpenTelemetry API does not provide Span ID API assert parent.get("spanID") == "10000" assert parent_meta.get("attributes") == "values" assert parent_meta.get("error.message") == "testing_end_span_options" - parent_metrics = interfaces.agent.get_span_metrics(parent, parent_span_format) + parent_metrics = interfaces.agent.get_span_metrics(parent) assert parent_metrics["_dd.top_level"] == 1.0 # Assert the child sent by the agent. # childName is no longer the operation name, rather the resource name # after remapping the OTel attributes to Datadog semantics - child, child_span_format = _get_span_by_resource(spans, "otel-name.dd-resource") - child_meta = interfaces.agent.get_span_meta(child, child_span_format) + child = _get_span_by_resource(spans, "otel-name.dd-resource") + child_meta = child.meta assert child.get("parentID") == parent.get("spanID") assert child.get("spanID") != "10000" assert child.get("duration") == "1000000000" @@ -59,21 +59,21 @@ def test_distributed_otel_trace(self): assert len(spans) >= 3, "Agent did not submit the spans we want!" # Assert the parent span sent by the agent. - parent, parent_span_format = _get_span_by_resource(spans, "root-otel-name.dd-resource") - assert interfaces.agent.get_span_name(parent, parent_span_format) == "internal" + parent = _get_span_by_resource(spans, "root-otel-name.dd-resource") + assert parent.get_span_name() == "internal" assert parent.get("parentID") is None - parent_metrics = interfaces.agent.get_span_metrics(parent, parent_span_format) + parent_metrics = interfaces.agent.get_span_metrics(parent) assert parent_metrics["_dd.top_level"] == 1.0 # Assert the Roundtrip child span sent by the agent, this span is created by an external OTel contrib package - roundtrip_span, roundtrip_span_format = _get_span_by_name(spans, "client.request") - assert interfaces.agent.get_span_name(roundtrip_span, roundtrip_span_format) == "client.request" - assert interfaces.agent.get_span_resource(roundtrip_span, roundtrip_span_format) == "HTTP GET" + roundtrip_span = _get_span_by_name(spans, "client.request") + assert roundtrip_span.get_span_name() == "client.request" + assert roundtrip_span.get_span_resource() == "HTTP GET" assert roundtrip_span.get("parentID") == parent.get("spanID") # Assert the Handler function child span sent by the agent. - handler_span, handler_span_format = _get_span_by_name(spans, "server.request") - assert interfaces.agent.get_span_resource(handler_span, handler_span_format) == "testOperation" + handler_span = _get_span_by_name(spans, "server.request") + assert handler_span.get_span_resource() == "testOperation" assert handler_span.get("parentID") == roundtrip_span.get("spanID") # Assert the spans received from the backend! @@ -81,19 +81,15 @@ def test_distributed_otel_trace(self): assert len(spans) == 3 -def _get_span_by_name( - spans: list[tuple[dict, TraceAgentPayloadFormat]], span_name: str -) -> tuple[dict, TraceAgentPayloadFormat]: - for s, span_format in spans: - if interfaces.agent.get_span_name(s, span_format) == span_name: - return s, span_format - return {}, TraceAgentPayloadFormat.legacy +def _get_span_by_name(spans: list[DataDogAgentSpan], span_name: str) -> DataDogAgentSpan: + for s in spans: + if s.get_span_name() == span_name: + return s + raise ValueError("Span not found") -def _get_span_by_resource( - spans: list[tuple[dict, TraceAgentPayloadFormat]], resource_name: str -) -> tuple[dict, TraceAgentPayloadFormat]: - for s, span_format in spans: - if interfaces.agent.get_span_resource(s, span_format) == resource_name: - return s, span_format - return {}, TraceAgentPayloadFormat.legacy +def _get_span_by_resource(spans: list[DataDogAgentSpan], resource_name: str) -> DataDogAgentSpan: + for s in spans: + if s.get_span_resource() == resource_name: + return s + raise ValueError("Span not found") diff --git a/tests/apm_tracing_e2e/test_process_tags.py b/tests/apm_tracing_e2e/test_process_tags.py index 7ed4d719705..b392b3f4dfc 100644 --- a/tests/apm_tracing_e2e/test_process_tags.py +++ b/tests/apm_tracing_e2e/test_process_tags.py @@ -17,7 +17,7 @@ def setup_tracing_process_tags_svc(self): def check_tracing_process_tags(self, validate_process_tags_func: Callable): # Get all the spans from the agent found = False - for data, _, _ in interfaces.agent.get_traces(self.req): + for data, _ in interfaces.agent.get_traces(self.req): # Check that the agent managed to extract the process tags from the first chunk if "idxTracerPayloads" in data["request"]["content"]: for payload in data["request"]["content"]["idxTracerPayloads"]: diff --git a/tests/apm_tracing_e2e/test_single_span.py b/tests/apm_tracing_e2e/test_single_span.py index 7c00c63e0a5..9189cdf99a8 100644 --- a/tests/apm_tracing_e2e/test_single_span.py +++ b/tests/apm_tracing_e2e/test_single_span.py @@ -5,8 +5,8 @@ SINGLE_SPAN_SAMPLING_MECHANISM_VALUE, SINGLE_SPAN_SAMPLING_RATE, SINGLE_SPAN_SAMPLING_MAX_PER_SEC, - TraceAgentPayloadFormat, ) +from utils.dd_types import DataDogAgentSpan @rfc("ATI-2419") @@ -32,12 +32,12 @@ def test_parent_span_is_single_span(self): assert len(spans) == 1, "Agent did not submit the spans we want!" # Assert the spans sent by the agent. - span, span_format = spans[0] - assert interfaces.agent.get_span_name(span, span_format) == "parent.span.single_span_submitted" + span = spans[0] + assert span.get_span_name() == "parent.span.single_span_submitted" assert span.get("parentID") is None - metrics = interfaces.agent.get_span_metrics(span, span_format) + metrics = interfaces.agent.get_span_metrics(span) assert metrics["_dd.top_level"] == 1.0 - _assert_single_span_metrics(span, span_format) + _assert_single_span_metrics(span) # Assert the spans received from the backend! backend_spans = interfaces.backend.assert_single_spans_exist(self.req) @@ -56,10 +56,10 @@ def test_child_span_is_single_span(self): assert len(spans) == 1, "Agent did not submit the spans we want!" # Assert the spans sent by the agent. - span, span_format = spans[0] - assert interfaces.agent.get_span_name(span, span_format) == "child.span.single_span_submitted" + span = spans[0] + assert span.get_span_name() == "child.span.single_span_submitted" assert span.get("parentID") is not None - _assert_single_span_metrics(span, span_format) + _assert_single_span_metrics(span) # Assert the spans received from the backend! backend_spans = interfaces.backend.assert_single_spans_exist(self.req) @@ -79,8 +79,8 @@ def _assert_single_span_event(event: dict, name: str, *, is_root: bool): assert len(parent_id) > 0, f"In a child span the parent_id should be specified. Actual: {parent_id}" -def _assert_single_span_metrics(span: dict, span_format: TraceAgentPayloadFormat): - metrics = interfaces.agent.get_span_metrics(span, span_format) +def _assert_single_span_metrics(span: DataDogAgentSpan): + metrics = interfaces.agent.get_span_metrics(span) assert metrics[SAMPLING_PRIORITY_KEY] == -1 # due to the global sampling rate = 0 assert metrics[SINGLE_SPAN_SAMPLING_RATE] == 1.0 assert metrics[SINGLE_SPAN_SAMPLING_MECHANISM] == SINGLE_SPAN_SAMPLING_MECHANISM_VALUE diff --git a/tests/appsec/rasp/test_api10.py b/tests/appsec/rasp/test_api10.py index a0cc862e200..b4e2c6aefb2 100644 --- a/tests/appsec/rasp/test_api10.py +++ b/tests/appsec/rasp/test_api10.py @@ -6,7 +6,7 @@ import urllib.parse from utils import features, weblog, interfaces, scenarios, rfc, context -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from tests.appsec.rasp.utils import ( find_series, @@ -28,7 +28,7 @@ class API10: TAGS_EXPECTED: list[tuple[str, str]] = [] TAGS_EXPECTED_METRIC: list[tuple[str, str]] = [] - def validate(self, span: DataDogSpan): + def validate(self, span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -44,7 +44,7 @@ def validate(self, span: DataDogSpan): return True - def validate_metric(self, span: DataDogSpan): + def validate_metric(self, span: DataDogLibrarySpan): for tag, expected in self.TAGS_EXPECTED_METRIC: # check also in meta to be safe assert tag in span["metrics"] or tag in span["meta"], f"Missing {tag} from span's meta/metrics" @@ -291,7 +291,7 @@ def setup_api10_res_body(self): "/external_request", data=json.dumps(self.BODY), headers={"Content-Type": "application/json"} ) - def validate_absence(self, span: DataDogSpan): + def validate_absence(self, span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -322,7 +322,7 @@ def setup_api10_res_body(self): "/external_request", data=json.dumps(self.BODY), headers={"Content-Type": "application/json"} ) - def validate_absence(self, span: DataDogSpan): + def validate_absence(self, span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None diff --git a/tests/appsec/test_asm_standalone.py b/tests/appsec/test_asm_standalone.py index dc9dc5ae1c8..68bff7a5c57 100644 --- a/tests/appsec/test_asm_standalone.py +++ b/tests/appsec/test_asm_standalone.py @@ -9,7 +9,7 @@ from utils.telemetry_utils import TelemetryUtils from utils._weblog import HttpResponse, _Weblog from utils import context, weblog, interfaces, scenarios, features, rfc, missing_feature, logger -from utils.dd_types import DataDogSpan, DataDogTrace, TraceLibraryPayloadFormat +from utils.dd_types import DataDogLibrarySpan, DataDogLibraryTrace, LibraryTraceFormat USER = "test" NEW_USER = "testnew" @@ -34,9 +34,9 @@ # - The value can be a string to assert the value of the tag # - The value can be a lambda function that will be used to assert the value of the tag (special case for _sampling_priority_v1) def assert_tags( - first_span: DataDogSpan, span: DataDogSpan, obj: str, expected_tags: dict[str, str | None | Callable] + first_span: DataDogLibrarySpan, span: DataDogLibrarySpan, obj: str, expected_tags: dict[str, str | None | Callable] ) -> bool: - def _assert_tags_value(span: DataDogSpan, obj: str, expected_tags: dict[str, str | None | Callable]): + def _assert_tags_value(span: DataDogLibrarySpan, obj: str, expected_tags: dict[str, str | None | Callable]): struct = span if obj is None else span[obj] for tag, value in expected_tags.items(): if value is None: @@ -61,8 +61,8 @@ def _assert_tags_value(span: DataDogSpan, obj: str, expected_tags: dict[str, str return False -def _assert_trace_id(trace: DataDogTrace, span: DataDogSpan, trace_id: int) -> None: - if trace.format == TraceLibraryPayloadFormat.v10: +def _assert_trace_id(trace: DataDogLibraryTrace, span: DataDogLibrarySpan, trace_id: int) -> None: + if trace.format == LibraryTraceFormat.v10: assert trace.trace_id_equals(trace_id) else: assert span.raw_span["trace_id"] == trace_id @@ -137,7 +137,7 @@ def setup_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_minus_ ) def fix_priority_lambda( - self, span: DataDogSpan, default_checks: dict[str, str | Callable | None] + self, span: DataDogLibrarySpan, default_checks: dict[str, str | Callable | None] ) -> dict[str, str | Callable | None]: if "_dd.appsec.s.req.headers" in span["meta"]: return { diff --git a/tests/appsec/test_automated_login_events.py b/tests/appsec/test_automated_login_events.py index 25bfcea0896..8a15060f8c3 100644 --- a/tests/appsec/test_automated_login_events.py +++ b/tests/appsec/test_automated_login_events.py @@ -12,7 +12,7 @@ from utils import scenarios from utils import weblog from utils.dd_constants import Capabilities, SamplingPriority -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan def login_data(username: str, password: str): @@ -529,7 +529,7 @@ def setup_login_success_headers(self): def test_login_success_headers(self): # Validate that all relevant headers are included on user login success on extended mode - def validate_login_success_headers(span: DataDogSpan): + def validate_login_success_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -549,7 +549,7 @@ def setup_login_failure_headers(self): def test_login_failure_headers(self): # Validate that all relevant headers are included on user login failure on extended mode - def validate_login_failure_headers(span: DataDogSpan): + def validate_login_failure_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -1023,7 +1023,7 @@ def setup_login_success_headers(self): def test_login_success_headers(self): # Validate that all relevant headers are included on user login success on extended mode - def validate_login_success_headers(span: DataDogSpan): + def validate_login_success_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -1043,7 +1043,7 @@ def setup_login_failure_headers(self): def test_login_failure_headers(self): # Validate that all relevant headers are included on user login failure on extended mode - def validate_login_failure_headers(span: DataDogSpan): + def validate_login_failure_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -1054,7 +1054,7 @@ def validate_login_failure_headers(span: DataDogSpan): interfaces.library.validate_one_span(self.r_hdr_failure, validator=validate_login_failure_headers) -def assert_priority(span: DataDogSpan, trace: list[DataDogSpan]): +def assert_priority(span: DataDogLibrarySpan, trace: list[DataDogLibrarySpan]): if "_sampling_priority_v1" not in span["metrics"]: # some tracers like java only send the priority in the first and last span of the trace assert trace[0]["metrics"].get("_sampling_priority_v1") == SamplingPriority.USER_KEEP @@ -1457,7 +1457,7 @@ def setup_login_success_headers(self): def test_login_success_headers(self): # Validate that all relevant headers are included on user login success on extended mode - def validate_login_success_headers(span: DataDogSpan): + def validate_login_success_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -1477,7 +1477,7 @@ def setup_login_failure_headers(self): def test_login_failure_headers(self): # Validate that all relevant headers are included on user login failure on extended mode - def validate_login_failure_headers(span: DataDogSpan): + def validate_login_failure_headers(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None diff --git a/tests/appsec/test_automated_payment_events.py b/tests/appsec/test_automated_payment_events.py index 8b3d3582b89..996308437c9 100644 --- a/tests/appsec/test_automated_payment_events.py +++ b/tests/appsec/test_automated_payment_events.py @@ -10,7 +10,7 @@ weblog, ) from utils._weblog import HttpResponse -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan WEBHOOK_SECRET = b"whsec_FAKE" @@ -32,7 +32,7 @@ def make_webhook_request(data: dict, secret: bytes = WEBHOOK_SECRET): ) -def assert_payment_event(request: HttpResponse, validator: Callable[[DataDogSpan], bool]): +def assert_payment_event(request: HttpResponse, validator: Callable[[DataDogLibrarySpan], bool]): assert request.status_code == 200 # make sure a Stripe object was returned by the Stripe SDK @@ -42,7 +42,7 @@ def assert_payment_event(request: HttpResponse, validator: Callable[[DataDogSpan assert body.get("livemode") # wrap validator for asserts common to all tests - def _validator(span: DataDogSpan): + def _validator(span: DataDogLibrarySpan): # discard non-root spans if span.get("parent_id") not in (0, None): return False @@ -58,7 +58,7 @@ def _validator(span: DataDogSpan): def assert_no_payment_event(request: HttpResponse, status_code: int): assert request.status_code == status_code - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert "appsec.events.payments.integration" not in span["meta"] interfaces.library.validate_all_spans(request, validator=validator) @@ -111,7 +111,7 @@ def setup_checkout_session(self): def test_checkout_session(self): """R1""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["appsec.events.payments.creation.id"] == "cs_FAKE" assert span["metrics"]["appsec.events.payments.creation.amount_total"] == 950 # 100 * 10 * 0.9 + 50 assert span["meta"]["appsec.events.payments.creation.client_reference_id"] == "GabeN" @@ -185,7 +185,7 @@ def setup_payment_intent(self): def test_payment_intent(self): """R2""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["appsec.events.payments.creation.id"] == "pi_FAKE" assert span["metrics"]["appsec.events.payments.creation.amount"] == 6969 assert span["meta"]["appsec.events.payments.creation.currency"] == "eur" @@ -216,7 +216,7 @@ def setup_payment_success(self): def test_payment_success(self): """R3""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["appsec.events.payments.success.id"] == "pi_FAKE" assert span["metrics"]["appsec.events.payments.success.amount"] == 420 assert span["meta"]["appsec.events.payments.success.currency"] == "eur" @@ -256,7 +256,7 @@ def setup_payment_failure(self): def test_payment_failure(self): """R4""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["appsec.events.payments.failure.id"] == "pi_FAKE" assert span["metrics"]["appsec.events.payments.failure.amount"] == 1337 assert span["meta"]["appsec.events.payments.failure.currency"] == "eur" @@ -294,7 +294,7 @@ def setup_payment_cancellation(self): def test_payment_cancellation(self): """R5""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["appsec.events.payments.cancellation.id"] == "pi_FAKE" assert span["metrics"]["appsec.events.payments.cancellation.amount"] == 1337 assert span["meta"]["appsec.events.payments.cancellation.cancellation_reason"] == "requested_by_customer" diff --git a/tests/appsec/test_blocking_addresses.py b/tests/appsec/test_blocking_addresses.py index 989ea9667be..9dc632c9582 100644 --- a/tests/appsec/test_blocking_addresses.py +++ b/tests/appsec/test_blocking_addresses.py @@ -16,11 +16,11 @@ features, HttpResponse, ) -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan def _assert_custom_event_tag_presence(expected_value: str): - def wrapper(span: DataDogSpan): + def wrapper(span: DataDogLibrarySpan): tag = "appsec.events.system_tests_appsec_event.value" assert tag in span["meta"], f"Can't find {tag} in span's meta" value = span["meta"][tag] @@ -31,7 +31,7 @@ def wrapper(span: DataDogSpan): def _assert_custom_event_tag_absence(): - def wrapper(span: DataDogSpan): + def wrapper(span: DataDogLibrarySpan): tag = "appsec.events.system_tests_appsec_event.value" assert tag not in span["meta"], f"Found {tag} in span's meta" return True diff --git a/tests/appsec/test_client_ip.py b/tests/appsec/test_client_ip.py index c3f9128e65f..8151b8128f8 100644 --- a/tests/appsec/test_client_ip.py +++ b/tests/appsec/test_client_ip.py @@ -3,7 +3,7 @@ # Copyright 2022 Datadog, Inc. from utils import weblog, interfaces, scenarios, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @scenarios.everything_disabled @@ -22,7 +22,7 @@ def setup_not_reported(self): def test_not_reported(self): """Test IP-related span tags are not reported when ASM is disabled""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): meta = span.get("meta", {}) assert "appsec.event" not in meta, "unexpected appsec event while appsec should be disabled" assert "http.client_ip" not in meta, "unexpected http.client_ip tag" diff --git a/tests/appsec/test_conf.py b/tests/appsec/test_conf.py index 18000a442a9..c095bee727a 100644 --- a/tests/appsec/test_conf.py +++ b/tests/appsec/test_conf.py @@ -5,7 +5,7 @@ from utils import weblog, context, interfaces, missing_feature, rfc, scenarios, features from utils.tools import nested_lookup -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan TELEMETRY_REQUEST_TYPE_GENERATE_METRICS = "generate-metrics" @@ -68,7 +68,7 @@ def setup_obfuscation_parameter_key(self): def test_obfuscation_parameter_key(self): """Test DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP""" - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) @@ -84,7 +84,7 @@ def setup_obfuscation_parameter_value(self): def test_obfuscation_parameter_value(self): """Test DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP""" - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET_WITH_HIDDEN_VALUE, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) @@ -110,7 +110,7 @@ def setup_partial_obfuscation_parameter_value(self): def test_partial_obfuscation_parameter_value(self): """Test DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP""" - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET_WITH_HIDDEN_VALUE, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) diff --git a/tests/appsec/test_event_tracking.py b/tests/appsec/test_event_tracking.py index d30415342ed..e02829e44b1 100644 --- a/tests/appsec/test_event_tracking.py +++ b/tests/appsec/test_event_tracking.py @@ -2,7 +2,7 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. from utils import weblog, interfaces, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from tests.appsec.utils import find_series HEADERS = { @@ -49,7 +49,7 @@ def setup_user_login_success_event(self): def test_user_login_success_event(self): # Call the user login success SDK and validate tags - def validate_user_login_success_tags(span: DataDogSpan): + def validate_user_login_success_tags(span: DataDogLibrarySpan): expected_tags = { "http.client_ip": "1.2.3.4", "usr.id": "system_tests_user", @@ -74,7 +74,7 @@ def setup_user_login_success_header_collection(self): def test_user_login_success_header_collection(self): # Validate that all relevant headers are included on user login success - def validate_user_login_success_header_collection(span: DataDogSpan) -> bool: + def validate_user_login_success_header_collection(span: DataDogLibrarySpan) -> bool: if span.get("parent_id") not in (0, None): return False @@ -118,7 +118,7 @@ def setup_user_login_failure_event(self): def test_user_login_failure_event(self): # Call the user login failure SDK and validate tags - def validate_user_login_failure_tags(span: DataDogSpan): + def validate_user_login_failure_tags(span: DataDogLibrarySpan): expected_tags = { "http.client_ip": "1.2.3.4", "appsec.events.users.login.failure.usr.id": "system_tests_user", @@ -144,7 +144,7 @@ def setup_user_login_failure_header_collection(self): def test_user_login_failure_header_collection(self): # Validate that all relevant headers are included on user login failure - def validate_user_login_failure_header_collection(span: DataDogSpan): + def validate_user_login_failure_header_collection(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -187,7 +187,7 @@ def setup_custom_event_event(self): def test_custom_event_event(self): # Call the user login failure SDK and validate tags - def validate_custom_event_tags(span: DataDogSpan): + def validate_custom_event_tags(span: DataDogLibrarySpan): expected_tags = { "http.client_ip": "1.2.3.4", "appsec.events.system_tests_event.track": "true", diff --git a/tests/appsec/test_event_tracking_v2.py b/tests/appsec/test_event_tracking_v2.py index 30ae06361b7..6406c88629d 100644 --- a/tests/appsec/test_event_tracking_v2.py +++ b/tests/appsec/test_event_tracking_v2.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. from utils import weblog, interfaces, features, scenarios -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from tests.appsec.utils import find_series from abc import ABC, abstractmethod @@ -45,7 +45,11 @@ def validate_metric_type_and_version(event_type: str, version: str, metric: dict def validate_tags_and_metadata( - span: DataDogSpan, prefix: str, expected_tags: dict, metadata: dict | None, unexpected_metadata: list[str] | None + span: DataDogLibrarySpan, + prefix: str, + expected_tags: dict, + metadata: dict | None, + unexpected_metadata: list[str] | None, ): if metadata is not None: for key, value in metadata.items(): @@ -71,7 +75,7 @@ class BaseUserLoginSuccessEventV2Tags: def get_user_login_success_tags_validator( self, login: str, user_id: str, metadata: dict | None = None, unexpected_metadata: list[str] | None = None ): - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): expected_tags = { "appsec.events.users.login.success.usr.login": login, "appsec.events.users.login.success.usr.id": user_id, @@ -215,7 +219,7 @@ def test_user_login_success_header_collection(self): assert self.r.status_code == 200 - def validate_user_login_success_header_collection(span: DataDogSpan): + def validate_user_login_success_header_collection(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -235,7 +239,7 @@ class Test_UserLoginSuccessEventV2_HeaderCollection_AppsecDisabled(BaseUserLogin def test_user_login_success_header_collection(self): assert self.r.status_code == 200 - def validate_user_login_success_header_collection(span: DataDogSpan): + def validate_user_login_success_header_collection(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -332,7 +336,7 @@ class BaseUserLoginFailureEventV2Tags: def get_user_login_failure_tags_validator( self, login: str, *, exists: bool, metadata: dict | None = None, unexpected_metadata: list[str] | None = None ): - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): expected_tags = { "appsec.events.users.login.failure.usr.login": login, "appsec.events.users.login.failure.usr.exists": "true" if exists else "false", @@ -472,7 +476,7 @@ class Test_UserLoginFailureEventV2_HeaderCollection_AppsecEnabled(BaseUserLoginF def test_user_login_failure_header_collection(self): assert self.r.status_code == 200 - def validate_user_login_failure_header_collection(span: DataDogSpan): + def validate_user_login_failure_header_collection(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -492,7 +496,7 @@ class Test_UserLoginFailureEventV2_HeaderCollection_AppsecDisabled(BaseUserLogin def test_user_login_failure_header_collection(self): assert self.r.status_code == 200 - def validate_user_login_failure_header_collection(span: DataDogSpan): + def validate_user_login_failure_header_collection(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None diff --git a/tests/appsec/test_identify.py b/tests/appsec/test_identify.py index e9dd40d4e7e..ab4014838cf 100644 --- a/tests/appsec/test_identify.py +++ b/tests/appsec/test_identify.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. from utils import weblog, interfaces, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @features.user_monitoring @@ -16,7 +16,7 @@ def setup_identify_tags_with_attack(self): def test_identify_tags_with_attack(self): # Send a random attack on the identify endpoint - should not affect the usr.id tag - def validate_identify_tags(span: DataDogSpan): + def validate_identify_tags(span: DataDogLibrarySpan): for tag in ["id", "name", "email", "session_id", "role", "scope"]: key = f"usr.{tag}" assert key in span["meta"], f"Can't find {key} in span's meta" diff --git a/tests/appsec/test_inferred_spans.py b/tests/appsec/test_inferred_spans.py index d017c074796..267757c6f6f 100644 --- a/tests/appsec/test_inferred_spans.py +++ b/tests/appsec/test_inferred_spans.py @@ -2,7 +2,7 @@ import time from utils import weblog, interfaces, scenarios, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan INFERRED_SPAN_NAMES = {"aws.apigateway", "aws.httpapi"} @@ -22,7 +22,7 @@ def test_lambda_inferred_span(self) -> None: assert lambda_span_appsec_data, "Expected non empty appsec data on aws.lambda span" - def validate_inferred_span(span: DataDogSpan) -> bool: + def validate_inferred_span(span: DataDogLibrarySpan) -> bool: if span.get("name") not in INFERRED_SPAN_NAMES: return False @@ -81,7 +81,7 @@ def test_proxy_inferred_span(self) -> None: assert service_entry_span_appsec_data, "Expected non empty appsec data on the weblog entry span" - def validate_inferred_span(span: DataDogSpan) -> bool: + def validate_inferred_span(span: DataDogLibrarySpan) -> bool: if span.get("name") != "aws.apigateway": return False diff --git a/tests/appsec/test_remote_config_rule_changes.py b/tests/appsec/test_remote_config_rule_changes.py index 32c87465ccc..004fde3fb04 100644 --- a/tests/appsec/test_remote_config_rule_changes.py +++ b/tests/appsec/test_remote_config_rule_changes.py @@ -5,7 +5,7 @@ import re from utils.dd_constants import Capabilities -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from tests.appsec.utils import find_series from utils import features from utils import interfaces @@ -163,14 +163,14 @@ def test_update_rules(self): expected_rules_version_tag = "_dd.appsec.event_rules.version" expected_version_regex = r"[0-9]+\.[0-9]+\.[0-9]+" - def validate_waf_rule_version_tag(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_waf_rule_version_tag(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 """Validate the mandatory event_rules.version tag is added to the request span having an attack""" meta = span["meta"] assert expected_rules_version_tag in meta, f"missing span meta tag `{expected_rules_version_tag}` in meta" assert re.match(expected_version_regex, meta[expected_rules_version_tag]) return True - def validate_waf_rule_version_tag_by_rc(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_waf_rule_version_tag_by_rc(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 """Validate the mandatory event_rules.version tag is added to the request span having an attack with expected rc version""" meta: dict = span["meta"] assert expected_rules_version_tag in meta, f"missing span meta tag `{expected_rules_version_tag}` in meta" diff --git a/tests/appsec/test_reports.py b/tests/appsec/test_reports.py index 7b5591bddca..e136f4df7dc 100644 --- a/tests/appsec/test_reports.py +++ b/tests/appsec/test_reports.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. from utils import weblog, interfaces, scenarios, rfc, features from utils._weblog import HttpResponse -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @features.security_events_metadata @@ -23,7 +23,7 @@ def check_http_code_legacy(event: dict): return True - def check_http_code(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def check_http_code(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 status_code = span["meta"]["http.status_code"] assert status_code == "404", f"404 should have been reported, not {status_code}" @@ -44,7 +44,7 @@ def setup_service(self): def test_service(self): """Appsec reports the service information""" - def _check_service_legacy(event: DataDogSpan): + def _check_service_legacy(event: DataDogLibrarySpan): name = event["context"]["service"]["name"] environment = event["context"]["service"]["environment"] assert name == "weblog", f"weblog should have been reported, not {name}" @@ -52,7 +52,7 @@ def _check_service_legacy(event: DataDogSpan): return True - def _check_service(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def _check_service(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 name = span.get("service") environment = span.get("meta", {}).get("env") assert name == "weblog", f"weblog should have been reported, not {name}" diff --git a/tests/appsec/test_shell_execution.py b/tests/appsec/test_shell_execution.py index e0163f35ba8..d6b3565df37 100644 --- a/tests/appsec/test_shell_execution.py +++ b/tests/appsec/test_shell_execution.py @@ -4,7 +4,7 @@ from utils import context, interfaces, weblog, features, irrelevant, rfc from utils._weblog import HttpResponse -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @rfc("https://docs.google.com/document/d/1YYxOB1nM032H-lgXrVml9mukMhF4eHVIzyK9H_PvrSY/edit#heading=h.o5gstqo08gu5") @@ -13,7 +13,7 @@ class Test_ShellExecution: """Test shell execution tracing""" @staticmethod - def fetch_command_execution_span(r: HttpResponse) -> DataDogSpan: + def fetch_command_execution_span(r: HttpResponse) -> DataDogLibrarySpan: assert r.status_code == 200 traces = [t for _, t in interfaces.library.get_traces(request=r)] diff --git a/tests/appsec/test_span_tags_headers.py b/tests/appsec/test_span_tags_headers.py index faa77af37dd..890cd653099 100644 --- a/tests/appsec/test_span_tags_headers.py +++ b/tests/appsec/test_span_tags_headers.py @@ -1,6 +1,6 @@ from utils import weblog, interfaces, features, scenarios, logger, rfc from utils._weblog import CaseInsensitiveDict -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan def validate_builder(headers: CaseInsensitiveDict, *, mandatory: bool = True): @@ -9,7 +9,7 @@ def validate_builder(headers: CaseInsensitiveDict, *, mandatory: bool = True): content_encoding = headers.get("Content-Encoding") content_language = headers.get("Content-Language") - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert (enabled := span["metrics"].get("_dd.appsec.enabled")) == 1.0, ( f"Expected _dd.appsec.enabled to be '1.0', got {enabled}" ) diff --git a/tests/appsec/test_trace_tagging.py b/tests/appsec/test_trace_tagging.py index 977fc275360..c2c01893775 100644 --- a/tests/appsec/test_trace_tagging.py +++ b/tests/appsec/test_trace_tagging.py @@ -9,7 +9,7 @@ features, ) from utils.dd_constants import Capabilities, SamplingPriority -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @features.appsec_trace_tagging_rules @@ -24,7 +24,7 @@ def setup_rule_with_attributes_no_keep_no_event(self): def test_rule_with_attributes_no_keep_no_event(self): """Test trace-tagging rule with attributes, no keep and no event""" - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -46,7 +46,7 @@ def setup_rule_with_attributes_keep_no_event(self): def test_rule_with_attributes_keep_no_event(self): """Test trace-tagging rule with attributes, sampling priority user_keep and no event""" - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -68,7 +68,7 @@ def setup_rule_with_attributes_keep_event(self): def test_rule_with_attributes_keep_event(self): """Test trace-tagging rule with attributes, sampling priority user_keep and an event""" - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None @@ -91,7 +91,7 @@ def setup_rule_with_attributes_no_keep_event(self): def test_rule_with_attributes_no_keep_event(self): """Test trace-tagging rule with attributes and an event, but no sampling priority change""" - def validate(span: DataDogSpan): + def validate(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): return None diff --git a/tests/appsec/test_traces.py b/tests/appsec/test_traces.py index 49e5050a1b4..c39ff4ce689 100644 --- a/tests/appsec/test_traces.py +++ b/tests/appsec/test_traces.py @@ -15,7 +15,7 @@ ) from utils.tools import nested_lookup from utils.dd_constants import SamplingPriority -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan RUNTIME_FAMILIES = ["nodejs", "ruby", "jvm", "dotnet", "go", "php", "python", "cpp"] @@ -41,7 +41,7 @@ def test_appsec_event_span_tags(self): _sampling_priority_v1 tags """ - def validate_appsec_event_span_tags(span: DataDogSpan): + def validate_appsec_event_span_tags(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return None @@ -172,7 +172,7 @@ def test_appsec_obfuscator_key(self): # Note that this value must contain an attack pattern in order to be part of the security event data # that is expected to be obfuscated. - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET_VALUE_WITH_SENSITIVE_KEY, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) @@ -224,7 +224,7 @@ def test_appsec_obfuscator_value(self): # The following payload will be sent as a raw encoded string via the request params # and matches an XSS attack. It contains an access token secret we shouldn't have in the event. - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.VALUE_WITH_SECRET, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) @@ -250,7 +250,7 @@ def test_appsec_obfuscator_key_with_custom_rules(self): # Note that this value must contain an attack pattern in order to be part of the security event data # that is expected to be obfuscated. - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET_VALUE_WITH_SENSITIVE_KEY, appsec_data, look_in_keys=True), ( "The security events contain the secret value that should be obfuscated" ) @@ -278,7 +278,7 @@ def test_appsec_obfuscator_cookies_with_custom_rules(self): # Note that this value must contain an attack pattern in order to be part of the security event data # that is expected to be obfuscated. - def validate_appsec_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_appsec_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 assert not nested_lookup(self.SECRET_VALUE_WITH_SENSITIVE_KEY_CUSTOM, appsec_data, look_in_keys=True), ( "Sensitive cookie is not obfuscated" ) @@ -306,11 +306,11 @@ def setup_header_collection(self): reason="The endpoint /headers is not implemented in the weblog", ) def test_header_collection(self): - def assert_header_in_span_meta(span: DataDogSpan, header: str): + def assert_header_in_span_meta(span: DataDogLibrarySpan, header: str): if header not in span["meta"]: raise Exception(f"Can't find {header} in span's meta") - def validate_response_headers(span: DataDogSpan): + def validate_response_headers(span: DataDogLibrarySpan): for header in ["content-type", "content-length", "content-language"]: assert_header_in_span_meta(span, f"http.response.headers.{header}") return True @@ -375,11 +375,11 @@ def setup_external_wafs_header_collection(self): def test_external_wafs_header_collection(self): """Collect external wafs request identifier and other security info when appsec is enabled.""" - def assert_header_in_span_meta(span: DataDogSpan, header: str): + def assert_header_in_span_meta(span: DataDogLibrarySpan, header: str): if header not in span["meta"]: raise Exception(f"Can't find {header} in span's meta") - def validate_request_headers(span: DataDogSpan): + def validate_request_headers(span: DataDogLibrarySpan): for header in [ "x-amzn-trace-id", "cloudfront-viewer-ja3-fingerprint", diff --git a/tests/appsec/test_user_blocking_full_denylist.py b/tests/appsec/test_user_blocking_full_denylist.py index 2bc2bb8bc0e..1b6a68a042c 100644 --- a/tests/appsec/test_user_blocking_full_denylist.py +++ b/tests/appsec/test_user_blocking_full_denylist.py @@ -1,5 +1,5 @@ from utils import interfaces, scenarios, weblog, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from .utils import BaseFullDenyListTest @@ -16,7 +16,7 @@ def setup_nonblocking_test(self): self.r_nonblock = weblog.get("/users", params={"user": self.NOT_BLOCKED_USER}) def test_nonblocking_test(self): - def validate_nonblock_user(span: DataDogSpan): + def validate_nonblock_user(span: DataDogLibrarySpan): assert span["meta"]["usr.id"] == self.NOT_BLOCKED_USER return True diff --git a/tests/appsec/waf/test_addresses.py b/tests/appsec/waf/test_addresses.py index e836f24f3d1..78f06fba7f0 100644 --- a/tests/appsec/waf/test_addresses.py +++ b/tests/appsec/waf/test_addresses.py @@ -4,7 +4,7 @@ import json import pytest from utils import weblog, bug, context, interfaces, missing_feature, rfc, scenarios, features, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @features.appsec_request_blocking @@ -398,7 +398,7 @@ def test_request_monitor_attack_directive(self): class Test_GrpcServerMethod: """Test as a custom rule until we have official rules for the address""" - def validate_span(self, span: DataDogSpan, appsec_data: dict): + def validate_span(self, span: DataDogLibrarySpan, appsec_data: dict): tag = "rpc.grpc.full_method" if tag not in span["meta"]: logger.info(f"Can't find '{tag}' in span's meta") diff --git a/tests/appsec/waf/test_blocking.py b/tests/appsec/waf/test_blocking.py index 46c86d546cd..252b19d5869 100644 --- a/tests/appsec/waf/test_blocking.py +++ b/tests/appsec/waf/test_blocking.py @@ -1,7 +1,7 @@ from pathlib import Path from utils import interfaces, bug, scenarios, weblog, rfc, features, context -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan from .test_blocking_security_response_id import ( is_valid_uuid4, extract_security_response_id_from_json, @@ -123,7 +123,7 @@ def test_blocking_appsec_blocked_tag(self): self.r_abt, pattern="Arachni/v", address="server.request.headers.no_cookies" ) - def validate_appsec_blocked(span: DataDogSpan): + def validate_appsec_blocked(span: DataDogLibrarySpan): if span.get("type") not in ("web", "serverless"): return None diff --git a/tests/appsec/waf/test_reports.py b/tests/appsec/waf/test_reports.py index 57fbdd18866..2773072058e 100644 --- a/tests/appsec/waf/test_reports.py +++ b/tests/appsec/waf/test_reports.py @@ -5,7 +5,7 @@ import json from utils import weblog, context, interfaces, irrelevant, scenarios, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan @features.support_in_app_waf_metrics_report @@ -27,7 +27,7 @@ def test_waf_monitoring(self): # Tags that are expected to be reported at least once at some point - def validate_waf_monitoring_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_waf_monitoring_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 """Validate the mandatory waf monitoring span tags are added to the request span having an attack""" meta = span["meta"] @@ -69,7 +69,7 @@ def test_waf_monitoring_once(self): expected_rules_monitoring_nb_errors_tag, ] - def validate_rules_monitoring_span_tags(span: DataDogSpan): + def validate_rules_monitoring_span_tags(span: DataDogLibrarySpan): """Validate the mandatory rules monitoring span tags are added to a request span at some point such as the first request or first attack. """ @@ -135,7 +135,7 @@ def test_waf_monitoring_once_rfc1025(self): # Tags that are expected to be reported at least once at some point expected_waf_version_tag = "_dd.appsec.waf.version" - def validate_rules_monitoring_span_tags(span: DataDogSpan): + def validate_rules_monitoring_span_tags(span: DataDogLibrarySpan): """Validate the mandatory rules monitoring span tags are added to a request span at some point such as the first request or first attack. """ @@ -166,7 +166,7 @@ def test_waf_monitoring_optional(self): expected_bindings_duration_metric = "_dd.appsec.waf.duration_ext" expected_metrics_tags = [expected_waf_duration_metric, expected_bindings_duration_metric] - def validate_waf_span_tags(span: DataDogSpan, appsec_data: dict): # noqa: ARG001 + def validate_waf_span_tags(span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG001 metrics = span["metrics"] for m in expected_metrics_tags: if m not in metrics: @@ -209,7 +209,7 @@ def test_waf_monitoring_errors(self): expected_nb_errors = 2 expected_error_details = {"missing key 'name'": ["missing-name"], "missing key 'tags'": ["missing-tags"]} - def validate_rules_monitoring_span_tags(span: DataDogSpan): + def validate_rules_monitoring_span_tags(span: DataDogLibrarySpan): """Validate the mandatory rules monitoring span tags are added to a request span at some point such as the first request or first attack. """ diff --git a/tests/debugger/test_debugger_code_origins.py b/tests/debugger/test_debugger_code_origins.py index 9ef74e95265..527ce39cba0 100644 --- a/tests/debugger/test_debugger_code_origins.py +++ b/tests/debugger/test_debugger_code_origins.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import tests.debugger.utils as debugger -from utils import scenarios, features, missing_feature, context, rfc, logger, interfaces +from utils import scenarios, features, missing_feature, context, rfc, logger @features.debugger_code_origins @@ -30,18 +30,18 @@ def test_code_origin_entry_present(self): self.assert_all_weblog_responses_ok() code_origins_entry_found = False - for span, span_format in self.all_spans: + for span in self.all_spans: # Web spans for the healthcheck should have code origins defined. try: - resource = interfaces.agent.get_span_resource(span, span_format) - resource_type = interfaces.agent.get_span_type(span, span_format) + resource = span.get_span_resource() + resource_type = span.get_span_type() except KeyError: # Some spans may not have resource or type fields, skip them continue logger.debug(span) if resource == "GET /healthcheck" and resource_type == "web": - meta = interfaces.agent.get_span_meta(span, span_format) + meta = span.meta code_origin_type = meta.get("_dd.code_origin.type", "") code_origins_entry_found = code_origin_type == "entry" diff --git a/tests/debugger/test_debugger_exception_replay.py b/tests/debugger/test_debugger_exception_replay.py index 57585052d4f..edc407db80c 100644 --- a/tests/debugger/test_debugger_exception_replay.py +++ b/tests/debugger/test_debugger_exception_replay.py @@ -10,8 +10,34 @@ import time from pathlib import Path from packaging import version -from utils import interfaces, scenarios, features, context, irrelevant, missing_feature, logger -from utils.dd_constants import TraceAgentPayloadFormat +from utils import scenarios, features, context, irrelevant, missing_feature, logger +from utils.dd_types import DataDogAgentSpan, AgentTraceFormat + + +def get_span_meta(span: dict, span_format: AgentTraceFormat) -> dict[str, str]: + """Returns the meta dictionary of a span according to its format""" + if span_format == AgentTraceFormat.legacy: + return span["meta"] + + if span_format == AgentTraceFormat.efficient_trace_payload_format: + # in the new format, metrics and meta are joined in attributes + return span["attributes"] + + raise ValueError(f"Unknown span format: {span_format}") + + +def set_span_attrs(span: dict, span_format: AgentTraceFormat, meta: dict[str, str]) -> None: + """Overwrites the span attributes of a span according to its format. + For legacy spans this means only the meta dictionary, + for efficient spans this means the entire attributes dictionary. + """ + if span_format == AgentTraceFormat.legacy: + span["meta"] = meta + + elif span_format == AgentTraceFormat.efficient_trace_payload_format: + span["attributes"] = meta + else: + raise ValueError(f"Unknown span format: {span_format}") def get_env_bool(env_var_name: str, *, default: bool = False) -> bool: @@ -88,35 +114,33 @@ def get_sort_key(content_tuple: tuple[str, dict]): return [snapshot for _, snapshot in filtered_contents] - def __filter_spans_by_snapshot_id(snapshots: list[dict]) -> dict[str, tuple[dict, TraceAgentPayloadFormat]]: + def __filter_spans_by_snapshot_id(snapshots: list[dict]) -> dict[str, DataDogAgentSpan]: filtered_spans = {} for snapshot in snapshots: snapshot_id = snapshot["id"] for spans in self.probe_spans.values(): - for span, span_format in spans: + for span in spans: snapshot_ids_in_span = { - key: value - for key, value in interfaces.agent.get_span_meta(span, span_format).items() - if key.endswith("snapshot_id") + key: value for key, value in span.meta.items() if key.endswith("snapshot_id") }.values() if snapshot_id in snapshot_ids_in_span: - filtered_spans[snapshot_id] = (span, span_format) + filtered_spans[snapshot_id] = span break return filtered_spans - def __filter_spans_by_span_id(contents: list[dict]) -> dict[str, tuple[dict, TraceAgentPayloadFormat]]: + def __filter_spans_by_span_id(contents: list[dict]) -> dict[str, DataDogAgentSpan]: filtered_spans = {} for content in contents: span_id = content.get("dd", {}).get("span_id") or content.get("dd.span_id") snapshot_id = content["debugger"]["snapshot"]["id"] for spans_list in self.probe_spans.values(): - for span, span_format in spans_list: + for span in spans_list: if span.get("spanID") == span_id: - filtered_spans[snapshot_id] = (span, span_format) + filtered_spans[snapshot_id] = span break return filtered_spans @@ -288,21 +312,21 @@ def __approve(snapshots: list): self.snapshots = snapshots __approve(snapshots) - def _validate_spans(self, test_name: str, spans: dict[str, tuple[dict, TraceAgentPayloadFormat]]): + def _validate_spans(self, test_name: str, spans: dict[str, DataDogAgentSpan]): def __scrub( - data: dict[str, tuple[dict, TraceAgentPayloadFormat]], - ) -> dict[str, tuple[dict, TraceAgentPayloadFormat]]: - scrubbed_spans: list[tuple[dict, TraceAgentPayloadFormat]] = [] + data: dict[str, DataDogAgentSpan], + ) -> dict[str, tuple[dict, AgentTraceFormat]]: + scrubbed_spans: list[tuple[dict, AgentTraceFormat]] = [] for value in data.values(): logger.debug(f"Value: {value}") - span_original, span_format = value + span_original, span_format = value.raw_span, value.trace.format span = copy.deepcopy(span_original) for scrub_key in ("traceID", "spanID", "parentID", "start", "duration", "metrics"): if scrub_key in span: span[scrub_key] = "" keys_to_remove = [] - span_meta = interfaces.agent.get_span_meta(span, span_format) + span_meta = get_span_meta(span, span_format) for meta_key, meta_value in span_meta.items(): if meta_key in { "_dd.appsec.fp.http.endpoint", @@ -326,25 +350,23 @@ def __scrub( for k in keys_to_remove: span_meta.pop(k, None) - interfaces.agent.set_span_attrs(span, span_format, dict(sorted(span_meta.items()))) + set_span_attrs(span, span_format, dict(sorted(span_meta.items()))) scrubbed_spans.append((span, span_format)) - sorted_spans = sorted( - scrubbed_spans, key=lambda x: interfaces.agent.get_span_meta(x[0], x[1])["error.type"] - ) + sorted_spans = sorted(scrubbed_spans, key=lambda x: get_span_meta(x[0], x[1])["error.type"]) - sorted_scrubbed_spans: dict[str, tuple[dict, TraceAgentPayloadFormat]] = {} + sorted_scrubbed_spans: dict[str, tuple[dict, AgentTraceFormat]] = {} # Assign scrubbed spans with unique snapshot labels for span_number, (span, span_format) in enumerate(sorted_spans): sorted_scrubbed_spans[f"snapshot_{span_number}"] = (dict(sorted(span.items())), span_format) return sorted_scrubbed_spans - def __approve(spans: dict[str, tuple[dict, TraceAgentPayloadFormat]]): + def __approve(spans: dict[str, tuple[dict, AgentTraceFormat]]): # Determine the payload format from spans to select the correct expected file # All spans in a test run should have the same format span_formats = {span_format for _, (_, span_format) in spans.items()} - is_v1_format = TraceAgentPayloadFormat.efficient_trace_payload_format in span_formats + is_v1_format = AgentTraceFormat.efficient_trace_payload_format in span_formats # Use different suffixes based on payload format expected_suffix = "spans_expected_v1" if is_v1_format else "spans_expected" @@ -374,7 +396,7 @@ def __approve(spans: dict[str, tuple[dict, TraceAgentPayloadFormat]]): missing_keys = [] # Use the interface helper to get meta/attributes based on format - span_meta = interfaces.agent.get_span_meta(span, span_format) + span_meta = get_span_meta(span, span_format) if "_dd.debug.error.exception_hash" not in span_meta: missing_keys.append("_dd.debug.error.exception_hash") @@ -392,11 +414,12 @@ def __approve(spans: dict[str, tuple[dict, TraceAgentPayloadFormat]]): assert spans, "Spans not found" - if not _SKIP_SCRUB: - spans = __scrub(spans) + if _SKIP_SCRUB: + self.spans = {key: (span.raw_span, span.trace.format) for key, span in spans.items()} + else: + self.spans = __scrub(spans) - self.spans = spans - __approve(spans) + __approve(self.spans) def _validate_recursion_snapshots(self, snapshots: list[dict], limit: int): assert len(snapshots) == limit + 1, ( @@ -449,8 +472,8 @@ def _validate_no_capture_reason(self, exception_key: str, expected_reason: str): found_expected_reason = False actual_reasons = [] - for span, span_format in spans_with_no_capture_reason: - meta = interfaces.agent.get_span_meta(span, span_format) + for span in spans_with_no_capture_reason: + meta = span.meta actual_reason = meta["_dd.debug.error.no_capture_reason"] actual_reasons.append(actual_reason) diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index 02eb05ea326..99668c8de32 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -12,7 +12,8 @@ from typing import TypedDict, Literal, Any from utils import interfaces, remote_config, weblog, context, logger -from utils.dd_constants import RemoteConfigApplyState as ApplyState, TraceAgentPayloadFormat +from utils.dd_constants import RemoteConfigApplyState as ApplyState +from utils.dd_types import DataDogAgentSpan # Agent paths _CONFIG_PATH = "/v0.7/config" @@ -100,8 +101,8 @@ class BaseDebuggerTest: probe_diagnostics: ProbeDiagnosticsCollection = {} probe_snapshots: dict[str, list[dict[str, Any]]] = {} - probe_spans: dict[str, list[tuple[dict[str, Any], TraceAgentPayloadFormat]]] = {} - all_spans: list[tuple[dict[str, Any], TraceAgentPayloadFormat]] = [] + probe_spans: dict[str, list[DataDogAgentSpan]] = {} + all_spans: list[DataDogAgentSpan] = [] symbols: list[dict[str, Any]] = [] start_time: int | None = None @@ -555,8 +556,8 @@ def _wait_for_no_capture_reason_span(self, data: dict): + "'" ) spans = interfaces.agent.get_spans_list() - for span, span_format in spans: - meta = interfaces.agent.get_span_meta(span, span_format) + for span in spans: + meta = span.meta if "_dd.debug.error.no_capture_reason" in meta: error_msg = meta.get("error.msg", "").lower() if self._error_message == error_msg: @@ -759,7 +760,7 @@ def _debugger_v2_input_snapshots_received(self): def _collect_spans(self): def _get_spans_hash(): - span_hash: dict[str, list[tuple[dict, TraceAgentPayloadFormat]]] = {} + span_hash: dict[str, list[DataDogAgentSpan]] = {} span_decoration_line_key = None if self.get_tracer()["language"] == "dotnet" or self.get_tracer()["language"] == "python": @@ -768,11 +769,11 @@ def _get_spans_hash(): span_decoration_line_key = "_dd.di.spandecorationargsandlocals.probe_id" spans_list = interfaces.agent.get_spans_list() - for span, span_format in spans_list: - self.all_spans.append((span, span_format)) + for span in spans_list: + self.all_spans.append(span) - span_name = interfaces.agent.get_span_name(span, span_format) - meta = interfaces.agent.get_span_meta(span, span_format) + span_name = span.get_span_name() + meta = span.meta is_span_decoration_method = span_name == "dd.dynamic.span" if is_span_decoration_method: @@ -780,7 +781,7 @@ def _get_spans_hash(): if probe_id: if probe_id not in span_hash: span_hash[probe_id] = [] - span_hash[probe_id].append((span, span_format)) + span_hash[probe_id].append(span) continue is_span_decoration_line = span_decoration_line_key in meta @@ -788,7 +789,7 @@ def _get_spans_hash(): probe_id = meta[span_decoration_line_key] if probe_id not in span_hash: span_hash[probe_id] = [] - span_hash[probe_id].append((span, span_format)) + span_hash[probe_id].append(span) continue has_exception_id = "_dd.debug.error.exception_id" in meta @@ -796,7 +797,7 @@ def _get_spans_hash(): exception_id = meta["_dd.debug.error.exception_id"] if exception_id not in span_hash: span_hash[exception_id] = [] - span_hash[exception_id].append((span, span_format)) + span_hash[exception_id].append(span) continue has_exception_capture_id = "_dd.debug.error.exception_capture_id" in meta @@ -812,13 +813,13 @@ def _get_spans_hash(): snapshot_id = meta[key] if snapshot_id not in span_hash: span_hash[snapshot_id] = [] - span_hash[snapshot_id].append((span, span_format)) + span_hash[snapshot_id].append(span) continue else: capture_id = meta["_dd.debug.error.exception_capture_id"] if capture_id not in span_hash: span_hash[capture_id] = [] - span_hash[capture_id].append((span, span_format)) + span_hash[capture_id].append(span) continue has_no_capture_reason = "_dd.debug.error.no_capture_reason" in meta @@ -826,7 +827,7 @@ def _get_spans_hash(): error_msg = meta.get("error.msg", "") if error_msg not in span_hash: span_hash[error_msg] = [] - span_hash[error_msg].append((span, span_format)) + span_hash[error_msg].append(span) continue # For Python, we need to look for spans with stack trace information diff --git a/tests/integrations/crossed_integrations/test_kafka.py b/tests/integrations/crossed_integrations/test_kafka.py index 3ba3282a37c..cc5eb40dc09 100644 --- a/tests/integrations/crossed_integrations/test_kafka.py +++ b/tests/integrations/crossed_integrations/test_kafka.py @@ -2,7 +2,7 @@ from utils import interfaces, scenarios, weblog, features, logger from utils.buddies import java_buddy, _Weblog as Weblog -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan def assert_trace_id_equality(a: str | int, b: str | int): @@ -26,7 +26,7 @@ class _BaseKafka: @classmethod def get_span( cls, interface: interfaces.LibraryInterfaceValidator, span_kind: str, topic: str - ) -> DataDogSpan | None: + ) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find traces with span kind: {span_kind} and topic: {topic} in {interface}") for data, trace in interface.get_traces(): @@ -47,7 +47,7 @@ def get_span( return None @staticmethod - def get_topic(span: DataDogSpan) -> str | None: + def get_topic(span: DataDogLibrarySpan) -> str | None: """Extracts the topic from a span by trying various fields""" topic = span["meta"].get("kafka.topic") # this is in python if topic is None: diff --git a/tests/integrations/crossed_integrations/test_kinesis.py b/tests/integrations/crossed_integrations/test_kinesis.py index ea2769de7c0..53acb4ee320 100644 --- a/tests/integrations/crossed_integrations/test_kinesis.py +++ b/tests/integrations/crossed_integrations/test_kinesis.py @@ -2,7 +2,7 @@ from utils.buddies import python_buddy, _Weblog as Weblog from utils import interfaces, scenarios, weblog, features, context, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan class _BaseKinesis: @@ -17,7 +17,7 @@ class _BaseKinesis: @classmethod def get_span( cls, interface: interfaces.LibraryInterfaceValidator, span_kind: list[str], stream: str, operation: str - ) -> DataDogSpan | None: + ) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find traces with span kind: {span_kind} and stream: {stream} in {interface}") for data, trace in interface.get_traces(): diff --git a/tests/integrations/crossed_integrations/test_rabbitmq.py b/tests/integrations/crossed_integrations/test_rabbitmq.py index f6ab357c1d4..e476dce2b9a 100644 --- a/tests/integrations/crossed_integrations/test_rabbitmq.py +++ b/tests/integrations/crossed_integrations/test_rabbitmq.py @@ -2,7 +2,7 @@ from utils.buddies import java_buddy, _Weblog as Weblog from utils import interfaces, scenarios, weblog, features, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan class _BaseRabbitMQ: @@ -25,7 +25,7 @@ def get_span( queue: str, exchange: str, operation: list[str], - ) -> DataDogSpan | None: + ) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find traces with span kind: {span_kind} and queue: {queue} in {interface}") for data, trace in interface.get_traces(): diff --git a/tests/integrations/crossed_integrations/test_sns_to_sqs.py b/tests/integrations/crossed_integrations/test_sns_to_sqs.py index dd0b6dfe408..0b47dd916ee 100644 --- a/tests/integrations/crossed_integrations/test_sns_to_sqs.py +++ b/tests/integrations/crossed_integrations/test_sns_to_sqs.py @@ -2,7 +2,7 @@ from utils.buddies import python_buddy, _Weblog as Weblog from utils import interfaces, scenarios, weblog, features, context, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan class _BaseSNS: @@ -24,7 +24,7 @@ def get_span( queue: str, topic: str, operation: str, - ) -> DataDogSpan | None: + ) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find traces with span kind: {span_kind} and queue: {queue} in {interface}") manual_span_found = False @@ -74,7 +74,7 @@ def get_span( return None @staticmethod - def get_queue(span: DataDogSpan) -> str | None: + def get_queue(span: DataDogLibrarySpan) -> str | None: """Extracts the queue from a span by trying various fields""" queue = span["meta"].get("queuename", None) # this is in nodejs, java, python @@ -90,7 +90,7 @@ def get_queue(span: DataDogSpan) -> str | None: return queue @staticmethod - def get_topic(span: DataDogSpan) -> str | None: + def get_topic(span: DataDogLibrarySpan) -> str | None: """Extracts the topic from a span by trying various fields""" topic = span["meta"].get("topicname", None) # this is in nodejs, java, python diff --git a/tests/integrations/crossed_integrations/test_sqs.py b/tests/integrations/crossed_integrations/test_sqs.py index 6867508b4f4..bb6547cf30f 100644 --- a/tests/integrations/crossed_integrations/test_sqs.py +++ b/tests/integrations/crossed_integrations/test_sqs.py @@ -2,7 +2,7 @@ from utils.buddies import python_buddy, java_buddy, _Weblog as Weblog from utils import interfaces, scenarios, weblog, features, context, irrelevant, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan class _BaseSQS: @@ -17,7 +17,7 @@ class _BaseSQS: @classmethod def get_span( cls, interface: interfaces.LibraryInterfaceValidator, span_kind: list[str], queue: str, operation: str - ) -> DataDogSpan | None: + ) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find traces with span kind: {span_kind} and queue: {queue} in {interface}") manual_span_found = False @@ -67,7 +67,7 @@ def get_span( return None @staticmethod - def get_queue(span: DataDogSpan) -> str | None: + def get_queue(span: DataDogLibrarySpan) -> str | None: """Extracts the queue from a span by trying various fields""" queue = span["meta"].get("queuename", None) # this is in nodejs, java, python diff --git a/tests/integrations/test_db_integrations_sql.py b/tests/integrations/test_db_integrations_sql.py index 7ff98fb118f..8ce18f00e42 100644 --- a/tests/integrations/test_db_integrations_sql.py +++ b/tests/integrations/test_db_integrations_sql.py @@ -2,8 +2,7 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. -from utils import context, bug, interfaces, missing_feature, scenarios, features, logger -from utils.dd_constants import TraceAgentPayloadFormat +from utils import context, bug, missing_feature, scenarios, features, logger from .utils import BaseDbIntegrationsTestClass @@ -20,33 +19,47 @@ def get_spans(self, excluded_operations: tuple[str, ...] = (), operations: list[ for db_operation, request in self.get_requests(excluded_operations, operations=operations): logger.debug(f"Start validation for {self.db_service}/{db_operation}") - logger.debug("Validating span generated by Datadog library") - yield db_operation, (self.get_span_from_tracer(request), TraceAgentPayloadFormat.legacy) + yield db_operation, self.get_span_from_tracer(request), self.get_span_from_agent(request) - logger.debug("Validating transmitted by Datadog agent") - yield db_operation, self.get_span_from_agent(request) + def get_spans_meta(self, excluded_operations: tuple[str, ...] = (), operations: list[str] | None = None): + """Get the spans meta from tracer and agent generated by all requests""" + + # yield the span from the tracer in first, as if it fails, there is a good chance that the one from the agent also fails + for db_operation, request in self.get_requests(excluded_operations, operations=operations): + logger.debug(f"Start validation for {self.db_service}/{db_operation}") + + yield db_operation, self.get_span_from_tracer(request).meta + yield db_operation, self.get_span_from_agent(request).meta # Tests methods def test_sql_traces(self, excluded_operations: tuple[str, ...] = ()): """After make the requests we check that we are producing sql traces""" - for _, (span, _span_format) in self.get_spans(excluded_operations): - assert span is not None + for _, lib_span, agent_span in self.get_spans(excluded_operations): + assert lib_span is not None + assert agent_span is not None def test_resource(self): """Usually the query""" - for db_operation, (span, span_format) in self.get_spans(excluded_operations=("procedure", "select_error")): - span_resource = interfaces.agent.get_span_resource(span, span_format) - assert db_operation in span_resource.lower() + for db_operation, lib_span, agent_span in self.get_spans(excluded_operations=("procedure", "select_error")): + assert db_operation in lib_span["resource"].lower() + assert db_operation in agent_span.get_span_resource().lower() def test_sql_success(self, excluded_operations: tuple[str, ...] = ()): """We check all sql launched for the app work""" - for db_operation, (span, span_format) in self.get_spans( + for db_operation, lib_span, agent_span in self.get_spans( excluded_operations=excluded_operations + ("select_error",) ): - if "error" in span and span["error"] != 0: - span_meta = interfaces.agent.get_span_meta(span, span_format) + if "error" in lib_span and lib_span["error"] != 0: + span_meta = lib_span.meta + logger.error(f"Error message: {span_meta.get('error.message')}") + logger.error(f"Error stack:\n{span_meta.get('error.stack')}") + + raise ValueError(f"Error found in {db_operation} operation, please check captured log call") + + if "error" in agent_span and agent_span["error"] != 0: + span_meta = agent_span.meta logger.error(f"Error message: {span_meta.get('error.message')}") logger.error(f"Error stack:\n{span_meta.get('error.stack')}") @@ -57,30 +70,26 @@ def test_db_type(self, excluded_operations: tuple[str, ...] = ()): Must be one of the available values: https://datadoghq.atlassian.net/wiki/spaces/APM/pages/2357395856/Span+attributes#db.system """ - for db_operation, (span, span_format) in self.get_spans(excluded_operations=excluded_operations): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(excluded_operations=excluded_operations): assert span_meta["db.type"] == self.db_service, f"Test is failing for {db_operation}" def test_db_name(self): """DEPRECATED!! Now it is db.instance. The name of the database being connected to. Database instance name.""" db_container = context.get_container_by_dd_integration_name(self.db_service) - for db_operation, (span, span_format) in self.get_spans(): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(): assert span_meta["db.name"] == db_container.db_instance, f"Test is failing for {db_operation}" def test_span_kind(self, excluded_operations: tuple[str, ...] = ()): """Describes the relationship between the Span, its parents, and its children in a Trace.""" - for _, (span, span_format) in self.get_spans(excluded_operations): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for _, span_meta in self.get_spans_meta(excluded_operations): assert span_meta["span.kind"] == "client" def test_runtime_id(self): """Unique identifier for the current process.""" - for db_operation, (span, span_format) in self.get_spans(): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(): assert span_meta["runtime-id"].strip(), f"Test is failing for {db_operation}" def test_db_system(self): @@ -88,47 +97,39 @@ def test_db_system(self): Must be one of the available values: https://datadoghq.atlassian.net/wiki/spaces/APM/pages/2357395856/Span+attributes#db.system """ - for db_operation, (span, span_format) in self.get_spans(): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(): assert span_meta["db.system"] == self.db_service, f"Test is failing for {db_operation}" def test_db_connection_string(self): """The connection string used to connect to the database.""" - for db_operation, (span, span_format) in self.get_spans(): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(): assert span_meta["db.connection_string"].strip(), f"Test is failing for {db_operation}" def test_db_user(self, excluded_operations: tuple[str, ...] = ()): """Username for accessing the database.""" db_container = context.get_container_by_dd_integration_name(self.db_service) - for _, (span, span_format) in self.get_spans(excluded_operations=excluded_operations): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for _, span_meta in self.get_spans_meta(excluded_operations=excluded_operations): assert span_meta["db.user"].casefold() == db_container.db_user.casefold() def test_db_instance(self, excluded_operations: tuple[str, ...] = ()): """The name of the database being connected to. Database instance name. Formerly db.name""" db_container = context.get_container_by_dd_integration_name(self.db_service) - for _, (span, span_format) in self.get_spans(excluded_operations=excluded_operations): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for _, span_meta in self.get_spans_meta(excluded_operations=excluded_operations): assert span_meta["db.instance"] == db_container.db_instance def test_db_statement_query(self): """Usually the query""" - for db_operation, (span, span_format) in self.get_spans(excluded_operations=("procedure", "select_error")): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(excluded_operations=("procedure", "select_error")): assert db_operation in span_meta["db.statement"].lower() def test_db_operation(self, excluded_operations: tuple[str, ...] = ()): """The name of the operation being executed""" - for db_operation, (span, span_format) in self.get_spans( - excluded_operations=excluded_operations + ("select_error",) - ): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(excluded_operations=excluded_operations + ("select_error",)): if db_operation == "procedure": assert any(substring in span_meta["db.operation"].lower() for substring in ["call", "exec"]), ( "db.operation span not found for procedure operation" @@ -139,8 +140,7 @@ def test_db_operation(self, excluded_operations: tuple[str, ...] = ()): def test_db_sql_table(self): """The name of the primary table that the operation is acting upon, including the database name (if applicable).""" - for db_operation, (span, span_format) in self.get_spans(excluded_operations=("procedure",)): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(excluded_operations=("procedure",)): assert span_meta["db.sql.table"].strip(), f"Test is failing for {db_operation}" def test_db_row_count(self): @@ -148,16 +148,14 @@ def test_db_row_count(self): This tag should only set for operations that retrieve stored data, such as GET operations and queries, excluding SET and other commands not returning data. """ - for _, (span, span_format) in self.get_spans(operations=["select"]): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for _, span_meta in self.get_spans_meta(operations=["select"]): assert int(span_meta["db.row_count"]) > 0, "Test is failing for select" def test_db_password(self, excluded_operations: tuple[str, ...] = ()): """The database password should not show in the traces""" db_container = context.get_container_by_dd_integration_name(self.db_service) - for db_operation, (span, span_format) in self.get_spans(excluded_operations=excluded_operations): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(excluded_operations=excluded_operations): for key in span_meta: if key not in [ "peer.hostname", @@ -176,13 +174,11 @@ def test_db_password(self, excluded_operations: tuple[str, ...] = ()): def test_db_jdbc_drive_classname(self): """The fully-qualified class name of the Java Database Connectivity (JDBC) driver used to connect.""" - for db_operation, (span, span_format) in self.get_spans(): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for db_operation, span_meta in self.get_spans_meta(): assert span_meta["db.jdbc.driver_classname"].strip(), f"Test is failing for {db_operation}" def test_error_message(self): - for _, (span, span_format) in self.get_spans(operations=["select_error"]): - span_meta = interfaces.agent.get_span_meta(span, span_format) + for _, span_meta in self.get_spans_meta(operations=["select_error"]): # A string representing the error message. assert span_meta["error.message"].strip() @@ -201,25 +197,23 @@ def test_not_obfuscate_query(self): def test_sql_query(self): """Usually the query""" for db_operation, request in self.get_requests(excluded_operations=["procedure", "select_error"]): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) - assert db_operation in span_meta["sql.query"].lower(), ( + span = self.get_span_from_agent(request) + assert db_operation in span.meta["sql.query"].lower(), ( f"sql.query span not found for operation {db_operation}" ) def test_obfuscate_query(self): """All queries come out obfuscated from agent""" for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) # We launch all queries with two parameters (from weblog) # Insert and procedure:These operations also receive two parameters, but are obfuscated as only one. if db_operation in ["insert", "procedure"]: - assert span_meta["sql.query"].count("?") == 1, ( + assert span.meta["sql.query"].count("?") == 1, ( f"The query is not properly obfuscated for operation {db_operation}" ) else: - assert span_meta["sql.query"].count("?") == 2, ( + assert span.meta["sql.query"].count("?") == 2, ( f"The query is not properly obfuscated for operation {db_operation}" ) @@ -261,8 +255,8 @@ def test_db_mssql_instance_name(self): This value should be set only if it's specified on the mssql connection string. """ - for db_operation, span in self.get_spans(): - assert span["meta"]["db.mssql.instance_name"].strip(), ( + for db_operation, span_meta in self.get_spans_meta(): + assert span_meta["db.mssql.instance_name"].strip(), ( f"db.mssql.instance_name must not be empty for operation {db_operation}" ) @@ -278,8 +272,7 @@ def test_db_user(self, excluded_operations: tuple[str, ...] = ()): # noqa: ARG0 def test_obfuscate_query(self): """All queries come out obfuscated from agent""" for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) # We launch all queries with two parameters (from weblog) if db_operation == "insert": expected_obfuscation_count = 1 @@ -290,9 +283,9 @@ def test_obfuscate_query(self): else: expected_obfuscation_count = 2 - observed_obfuscation_count = span_meta["sql.query"].count("?") + observed_obfuscation_count = span.meta["sql.query"].count("?") assert observed_obfuscation_count == expected_obfuscation_count, ( - f"The mssql query is not properly obfuscated for operation {db_operation}, expecting {expected_obfuscation_count} obfuscation(s), found {observed_obfuscation_count}:\n {span_meta['sql.query']}" + f"The mssql query is not properly obfuscated for operation {db_operation}, expecting {expected_obfuscation_count} obfuscation(s), found {observed_obfuscation_count}:\n {span.meta['sql.query']}" ) @bug(context.library == "python" and context.weblog_variant in ("flask-poc", "uds-flask"), reason="APMAPI-1058") diff --git a/tests/integrations/test_dbm.py b/tests/integrations/test_dbm.py index a689c15f038..8c49503a2eb 100644 --- a/tests/integrations/test_dbm.py +++ b/tests/integrations/test_dbm.py @@ -8,7 +8,7 @@ from utils import weblog, interfaces, context, scenarios, features, irrelevant, bug, logger from utils._weblog import HttpResponse -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan def remove_traceparent(s: str) -> str: @@ -55,7 +55,7 @@ def weblog_trace_payload(self): ] ) - def _get_db_span(self, response: HttpResponse) -> DataDogSpan: + def _get_db_span(self, response: HttpResponse) -> DataDogLibrarySpan: assert response.status_code == 200, f"Request: {context.scenario.name} wasn't successful." spans = [] @@ -84,11 +84,11 @@ def _assert_spans_are_untagged(self): for request in self.requests: self._assert_span_is_untagged(self._get_db_span(request)) - def _assert_span_is_untagged(self, span: DataDogSpan) -> None: + def _assert_span_is_untagged(self, span: DataDogLibrarySpan) -> None: meta = span.get("meta", {}) assert self.META_TAG not in meta, f"{self.META_TAG} found in span meta: {json.dumps(span.raw_span, indent=2)}" - def _assert_span_is_tagged(self, span: DataDogSpan) -> None: + def _assert_span_is_tagged(self, span: DataDogLibrarySpan) -> None: meta = span.get("meta", {}) assert self.META_TAG in meta, f"{self.META_TAG} not found in span meta: {json.dumps(span.raw_span, indent=2)}" tag_value = meta.get(self.META_TAG) diff --git a/tests/integrations/test_inferred_proxy.py b/tests/integrations/test_inferred_proxy.py index 7166fe30368..3b2b36d4cef 100644 --- a/tests/integrations/test_inferred_proxy.py +++ b/tests/integrations/test_inferred_proxy.py @@ -3,7 +3,7 @@ from typing import Literal from utils import weblog, scenarios, features, interfaces, logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan DISTRIBUTED_TRACE_ID = 1 DISTRIBUTED_PARENT_ID = 2 @@ -115,7 +115,7 @@ def test_api_gateway_inferred_span_creation_error(self): ) -def get_span(interface: interfaces.LibraryInterfaceValidator, resource: str) -> DataDogSpan | None: +def get_span(interface: interfaces.LibraryInterfaceValidator, resource: str) -> DataDogLibrarySpan | None: logger.debug(f"Trying to find API Gateway span for interface: {interface}") for data, trace in interface.get_traces(): @@ -138,7 +138,7 @@ def get_span(interface: interfaces.LibraryInterfaceValidator, resource: str) -> def assert_api_gateway_span( test_case: _BaseTestCase, - span: DataDogSpan, + span: DataDogLibrarySpan, path: str, status_code: str, *, @@ -216,7 +216,7 @@ def mandatory_tags_validator_factory( else: expected_component = "aws-httpapi" - def validate_api_gateway_span(span: DataDogSpan) -> bool: + def validate_api_gateway_span(span: DataDogLibrarySpan) -> bool: if span.get("metrics", {}).get("_dd.inferred_span") != 1: return False @@ -300,7 +300,7 @@ def optional_tags_validator_factory(proxy: Literal["aws.apigateway", "aws.httpap else: expected_arn = "arn:aws:apigateway:eu-west-3::/apis/a1b2c3d4e5f" - def validate_api_gateway_span(span: DataDogSpan) -> bool: + def validate_api_gateway_span(span: DataDogLibrarySpan) -> bool: if span.get("metrics", {}).get("_dd.inferred_span") != 1: return False diff --git a/tests/integrations/test_open_telemetry.py b/tests/integrations/test_open_telemetry.py index a9b154d49f4..d21f4e77383 100644 --- a/tests/integrations/test_open_telemetry.py +++ b/tests/integrations/test_open_telemetry.py @@ -1,4 +1,4 @@ -from utils import context, features, interfaces, irrelevant, missing_feature, scenarios, logger +from utils import context, features, irrelevant, missing_feature, scenarios, logger from .utils import BaseDbIntegrationsTestClass import json @@ -17,16 +17,16 @@ def test_properties(self): for db_operation, request in self.get_requests(): logger.info(f"Validating {self.db_service}/{db_operation}") - span, span_format = self.get_span_from_agent(request) + span = self.get_span_from_agent(request) assert span is not None, f"Span is not found for {db_operation}" # DEPRECATED!! Now it is db.instance. The name of the database being connected to. Database instance name. - span_meta = interfaces.agent.get_span_meta(span, span_format) + span_meta = span.meta assert span_meta["db.name"] == db_container.db_instance # Describes the relationship between the Span, its parents, and its children in a Trace. - assert interfaces.agent.get_span_kind(span, span_format) in ("client", "SPAN_KIND_CLIENT") + assert span.get_span_kind() in ("client", "SPAN_KIND_CLIENT") # An identifier for the database management system (DBMS) product being used. Formerly db.type # Must be one of the available values: @@ -54,21 +54,21 @@ def test_properties(self): def test_resource(self): """Usually the query""" for db_operation, request in self.get_requests(excluded_operations=["procedure", "select_error"]): - span, span_format = self.get_span_from_agent(request) - assert db_operation in interfaces.agent.get_span_resource(span, span_format).lower() + span = self.get_span_from_agent(request) + assert db_operation in span.get_span_resource().lower() def test_db_connection_string(self): """The connection string used to connect to the database.""" for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta assert span_meta["db.connection_string"].strip(), f"Test is failing for {db_operation}" def test_db_operation(self): """The name of the operation being executed""" for db_operation, request in self.get_requests(excluded_operations=["select_error"]): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta if db_operation == "procedure": assert any(substring in span_meta["db.operation"].lower() for substring in ["call", "exec"]), ( @@ -84,19 +84,19 @@ def test_db_operation(self): def test_db_sql_table(self): """The name of the primary table that the operation is acting upon, including the database name (if applicable).""" for db_operation, request in self.get_requests(excluded_operations=["procedure"]): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta assert span_meta["db.sql.table"].strip(), f"Test is failing for {db_operation}" def test_error_message(self): """A string representing the error message.""" - span, span_format = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) + span_meta = span.meta assert len(span_meta["error.msg"].strip()) != 0 def test_error_type_and_stack(self): - span, span_format = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) + span_meta = span.meta # A string representing the type of the error assert span_meta["error.type"].strip() @@ -107,8 +107,8 @@ def test_error_type_and_stack(self): def test_error_exception_event(self): """New version of test_error_type_and_stack() starting agent@7.75.0""" - span, span_format = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(self.requests[self.db_service]["select_error"]) + span_meta = span.meta events = json.loads(span_meta["events"]) exception_events = [event for event in events if event["name"] == "exception"] assert len(exception_events) > 0 @@ -120,8 +120,8 @@ def test_error_exception_event(self): def test_obfuscate_query(self): """All queries come out obfuscated from agent""" for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta if db_operation in ["update", "delete", "procedure", "select_error", "select"]: assert span_meta["db.statement"].count("?") == 2, ( f"The query is not properly obfuscated for operation {db_operation}" @@ -134,14 +134,14 @@ def test_obfuscate_query(self): def test_sql_success(self): """We check all sql launched for the app work""" for _, request in self.get_requests(excluded_operations=["select_error"]): - span, _ = self.get_span_from_agent(request) + span = self.get_span_from_agent(request) assert "error" not in span or span["error"] == 0 def test_db_statement_query(self): """Usually the query""" for db_operation, request in self.get_requests(excluded_operations=["procedure", "select_error"]): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta assert db_operation in span_meta["db.statement"].lower(), ( f"{db_operation} not found in {span_meta['db.statement']}" ) @@ -182,8 +182,8 @@ def test_db_mssql_instance_name(self): This value should be set only if it's specified on the mssql connection string. """ for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta assert span_meta["db.mssql.instance_name"].strip(), ( f"db.mssql.instance_name must not be empty for operation {db_operation}" ) @@ -191,8 +191,8 @@ def test_db_mssql_instance_name(self): def test_db_operation(self): """The name of the operation being executed. Mssql and Open Telemetry doesn't report this span when we call to procedure""" for db_operation, request in self.get_requests(excluded_operations=["select_error", "procedure"]): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta # db.operation span is not generating by Open Telemetry when we call to procedure or we have a syntax error on the SQL if db_operation not in ["select_error", "procedure"]: assert db_operation.lower() in span_meta["db.operation"].lower(), f"Test is failing for {db_operation}" @@ -206,8 +206,8 @@ def test_db_connection_string(self): def test_obfuscate_query(self): """All queries come out obfuscated from agent""" for db_operation, request in self.get_requests(): - span, span_format = self.get_span_from_agent(request) - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = self.get_span_from_agent(request) + span_meta = span.meta if db_operation in ["insert", "select"]: expected_obfuscation_count = 3 diff --git a/tests/integrations/utils.py b/tests/integrations/utils.py index 4cb047e0556..db0ce4bfa90 100644 --- a/tests/integrations/utils.py +++ b/tests/integrations/utils.py @@ -3,7 +3,7 @@ import struct from utils import weblog, interfaces, logger, HttpResponse -from utils.dd_constants import TraceAgentPayloadFormat +from utils.dd_types import DataDogAgentSpan, DataDogLibrarySpan class BaseDbIntegrationsTestClass: @@ -84,7 +84,7 @@ def get_requests( yield db_operation, request @staticmethod - def get_span_from_tracer(weblog_request: HttpResponse) -> dict: + def get_span_from_tracer(weblog_request: HttpResponse) -> DataDogLibrarySpan: for _, _, span in interfaces.library.get_spans(weblog_request): logger.info(f"Span found with trace id: {span['trace_id']} and span id: {span['span_id']}") @@ -109,15 +109,15 @@ def get_span_from_tracer(weblog_request: HttpResponse) -> dict: raise ValueError(f"Span is not found for {weblog_request.request.url}") @staticmethod - def get_span_from_agent(weblog_request: HttpResponse) -> tuple[dict, TraceAgentPayloadFormat]: - for data, chunk, chunk_format in interfaces.agent.get_traces(weblog_request): - trace_id = interfaces.agent.get_trace_id(chunk, chunk_format) + def get_span_from_agent(weblog_request: HttpResponse) -> DataDogAgentSpan: + for data, chunk in interfaces.agent.get_traces(weblog_request): + trace_id = chunk.trace_id_as_int logger.debug(f"Chunk found: trace id={trace_id}; ({data['log_filename']})") # iterate over everything to be sure to miss nothing - for span in chunk["spans"]: + for span in chunk.spans: logger.debug(f"Checking if span {span['spanID']} could match") - span_type = interfaces.agent.get_span_type(span, chunk_format) + span_type = span.get_span_type() if span_type not in ("sql", "db"): logger.debug(f"Wrong type:{span_type}, continue...") # no way it's the span we're looking for @@ -125,26 +125,26 @@ def get_span_from_agent(weblog_request: HttpResponse) -> tuple[dict, TraceAgentP # workaround to avoid conflicts on connection check on mssql # workaround to avoid conflicts on connection check on mssql + nodejs + opentelemetry (there is a bug in the sql obfuscation) - span_resource = interfaces.agent.get_span_resource(span, chunk_format) + span_resource = span.get_span_resource() if span_resource in ("SELECT ?", "SELECT 1;"): logger.debug(f"Wrong resource:{span_resource}, continue...") continue # workaround to avoid conflicts on postgres + nodejs + opentelemetry - span_name = interfaces.agent.get_span_name(span, chunk_format) + span_name = span.get_span_name() if span_name == "pg.connect": logger.debug(f"Wrong name:{span_name}, continue...") continue # workaround to avoid conflicts on mssql + nodejs + opentelemetry - span_meta = interfaces.agent.get_span_meta(span, chunk_format) + span_meta = span.meta if span_meta.get("db.statement") == "SELECT 1;": logger.debug(f"Wrong db.statement:{span_meta.get('db.statement')}, continue...") continue logger.info(f"Span type==sql found: spanId={span['spanID']}") - return span, chunk_format + return span raise ValueError(f"Span is not found for {weblog_request.request.url}") diff --git a/tests/serverless/span_pointers/utils.py b/tests/serverless/span_pointers/utils.py index 158a94f33ea..cd753e19f62 100644 --- a/tests/serverless/span_pointers/utils.py +++ b/tests/serverless/span_pointers/utils.py @@ -3,7 +3,7 @@ from typing import NewType from utils import logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan PointerHash = NewType("PointerHash", str) @@ -30,14 +30,14 @@ def standard_hashing_function(elements: list[bytes]) -> PointerHash: def make_single_span_link_validator( resource: str, pointer_kind: str, pointer_direction: str, pointer_hash: PointerHash -) -> Callable[[DataDogSpan], bool]: +) -> Callable[[DataDogLibrarySpan], bool]: """Make a validator function for use with interfaces.library.validate_spans. The validator checks that there is one and only one span pointer for the pointer_kind and pointer_direction and that its hash matches the pointer_hash. """ - def validator(span: DataDogSpan) -> bool: + def validator(span: DataDogLibrarySpan) -> bool: logger.debug("checking span: %s", span) if "span_links" not in span: diff --git a/tests/test_config_consistency.py b/tests/test_config_consistency.py index bee197f2604..83afc395c7b 100644 --- a/tests/test_config_consistency.py +++ b/tests/test_config_consistency.py @@ -44,10 +44,10 @@ def test_status_code_400(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of spans" - span, span_format = spans[0] + span = spans[0] - assert interfaces.agent.get_span_type(span, span_format) == "web" - span_meta = interfaces.agent.get_span_meta(span, span_format) + assert span.get_span_type() == "web" + span_meta = span.meta assert span_meta["http.status_code"] == "400" assert "error" not in span or span["error"] == 0 @@ -60,9 +60,9 @@ def test_status_code_500(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of spans" - span, span_format = spans[0] + span = spans[0] - span_meta = interfaces.agent.get_span_meta(span, span_format) + span_meta = span.meta assert span_meta["http.status_code"] == "500" assert span["error"] @@ -81,9 +81,9 @@ def test_status_code_200(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of chunks" - span, span_format = spans[0] - assert interfaces.agent.get_span_type(span, span_format) == "web" - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = spans[0] + assert span.get_span_type() == "web" + span_meta = span.meta assert span_meta["http.status_code"] == "200" assert span["error"] @@ -96,9 +96,9 @@ def test_status_code_202(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of chunks" - span, span_format = spans[0] - assert interfaces.agent.get_span_type(span, span_format) == "web" - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = spans[0] + assert span.get_span_type() == "web" + span_meta = span.meta assert span_meta.get("http.status_code") == "202" assert span.get("error") @@ -410,8 +410,7 @@ def test_specified_service_name(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, f"Agent received the incorrect amount of spans, Spans: {spans}" - span_format = spans[0][1] - assert interfaces.agent.get_span_service(spans[0][0], span_format) == "service_test" + assert spans[0].get_span_service() == "service_test" @scenarios.default @@ -426,10 +425,10 @@ def test_default_service_name(self): interfaces.library.assert_trace_exists(self.r) spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of spans" - span, span_format = spans[0] + span = spans[0] assert ( - interfaces.agent.get_span_service(span, span_format) != "service_test" + span.get_span_service() != "service_test" ) # in default scenario, DD_SERVICE is set to "weblog" in the dockerfile; this is a temp fix to test that it is not the value we manually set in the specific scenario diff --git a/tests/test_data_integrity.py b/tests/test_data_integrity.py index ee9953759c8..9ca5a24c698 100644 --- a/tests/test_data_integrity.py +++ b/tests/test_data_integrity.py @@ -6,8 +6,8 @@ import string from utils import weblog, interfaces, context, rfc, missing_feature, features, scenarios, logger -from utils.dd_constants import SamplingPriority, TraceAgentPayloadFormat -from utils.dd_types import DataDogTrace, TraceLibraryPayloadFormat +from utils.dd_constants import SamplingPriority +from utils.dd_types import DataDogLibraryTrace, LibraryTraceFormat, AgentTraceFormat from utils.cgroup_info import get_container_id @@ -215,7 +215,7 @@ def test_traces_coherence(self): assert isinstance(trace_id, int) assert trace_id > 0 - if trace.format in (TraceLibraryPayloadFormat.v04, TraceLibraryPayloadFormat.v05): + if trace.format in (LibraryTraceFormat.v04, LibraryTraceFormat.v05): for span in trace: assert span.raw_span["trace_id"] == trace_id @@ -229,19 +229,17 @@ def test_agent_do_not_drop_traces(self): # get list of trace ids reported by the agent trace_ids_reported_by_agent = set[int]() - for _, chunk, chunk_format in interfaces.agent.get_traces(): - if chunk_format == TraceAgentPayloadFormat.efficient_trace_payload_format: + for _, chunk in interfaces.agent.get_traces(): + if chunk.format == AgentTraceFormat.efficient_trace_payload_format: # the chunk TraceID is a hex encoded string like "0x69274AA50000000068F1C3D5F2D1A9B0" # We need to convert it to an integer taking only the lower 64 bits # Note that this ignores the upper 64 bits, but this is fine for just verifying that the trace is reported for our test - trace_id = int(chunk["traceID"], 16) & 0xFFFFFFFFFFFFFFFF + trace_id = chunk.trace_id_as_int trace_ids_reported_by_agent.add(trace_id) - elif chunk_format == TraceAgentPayloadFormat.legacy: - for span in chunk["spans"]: - trace_ids_reported_by_agent.add(int(span["traceID"])) - break + elif chunk.format == AgentTraceFormat.legacy: + trace_ids_reported_by_agent.add(chunk.trace_id_as_int) - def get_span_with_sampling_data(trace: DataDogTrace): + def get_span_with_sampling_data(trace: DataDogLibraryTrace): # The root span is not necessarily the span wherein the sampling priority can be found. # If present, the root will take precedence, and otherwise the first span with the # sampling priority tag will be returned. This is the same logic found on the trace-agent. diff --git a/tests/test_distributed.py b/tests/test_distributed.py index d6e4c449e0b..6243f2ef926 100644 --- a/tests/test_distributed.py +++ b/tests/test_distributed.py @@ -4,9 +4,8 @@ import json from utils import weblog, interfaces, scenarios, features, bug, context -from utils.dd_constants import TraceAgentPayloadFormat from utils.docker_fixtures.spec.trace import SAMPLING_PRIORITY_KEY, ORIGIN -from utils.dd_types import DataDogSpan, TraceLibraryPayloadFormat +from utils.dd_types import DataDogLibrarySpan, LibraryTraceFormat @scenarios.trace_propagation_style_w3c @@ -211,8 +210,8 @@ def test_span_links_omit_tracestate_from_conflicting_contexts(self): assert link1.get("tracestate") is None -def _retrieve_span_links(span: DataDogSpan): - if span.trace.format == TraceLibraryPayloadFormat.v10: +def _retrieve_span_links(span: DataDogLibrarySpan): + if span.trace.format == LibraryTraceFormat.v10: return span.raw_span["attributes"].get("_dd.span_links") if span.get("span_links") is not None: @@ -266,15 +265,15 @@ def test_synthetics(self): traces = list(interfaces.agent.get_traces(self.r)) assert len(traces) == 1, "Agent received the incorrect amount of traces" - _, trace, trace_format = traces[0] - self.assert_trace_id_equals(trace, trace_format, "1234567890") + _, trace = traces[0] + assert trace.trace_id_as_int == 1234567890 spans = list(interfaces.agent.get_spans(self.r)) assert len(spans) == 1, "Agent received the incorrect amount of spans" - _, span, span_format = spans[0] + _, span = spans[0] assert "parentID" not in span or span.get("parentID") == 0 or span.get("parentID") is None - meta = interfaces.agent.get_span_meta(span, span_format) - metrics = interfaces.agent.get_span_metrics(span, span_format) + meta = span.meta + metrics = interfaces.agent.get_span_metrics(span) assert meta[ORIGIN] == "synthetics" assert metrics[SAMPLING_PRIORITY_KEY] == 1 @@ -293,28 +292,15 @@ def test_synthetics_browser(self): interfaces.library.assert_trace_exists(self.r) traces = list(interfaces.agent.get_traces(self.r)) assert len(traces) == 1, "Agent received the incorrect amount of traces" - _, trace, trace_format = traces[0] - self.assert_trace_id_equals(trace, trace_format, "1234567891") + _, trace = traces[0] + assert trace.trace_id_as_int == 1234567891 spans = list(interfaces.agent.get_spans(self.r)) assert len(spans) == 1, "Agent received the incorrect amount of spans" - _, span, span_format = spans[0] + _, span = spans[0] assert "parentID" not in span or span.get("parentID") == 0 or span.get("parentID") is None - meta = interfaces.agent.get_span_meta(span, span_format) - metrics = interfaces.agent.get_span_metrics(span, span_format) + meta = span.meta + metrics = interfaces.agent.get_span_metrics(span) assert meta[ORIGIN] == "synthetics-browser" assert metrics[SAMPLING_PRIORITY_KEY] == 1 - - @staticmethod - def assert_trace_id_equals(trace: dict, trace_format: TraceAgentPayloadFormat, expected_trace_id: str) -> None: - if trace_format == TraceAgentPayloadFormat.legacy: - actual_trace_id = str(trace["spans"][0]["traceID"]) - assert expected_trace_id == actual_trace_id - elif trace_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - actual_trace_id = str(trace["traceID"]) - # In efficient trace payload format, traceID is in hex format - actual_trace_id = str(int(actual_trace_id, 16)) - assert actual_trace_id == expected_trace_id - else: - raise ValueError(f"Unknown span format: {trace_format}") diff --git a/tests/test_graphql.py b/tests/test_graphql.py index 33cea269098..efd67ece8f7 100644 --- a/tests/test_graphql.py +++ b/tests/test_graphql.py @@ -12,7 +12,7 @@ features, scenarios, ) -from utils.dd_types import DataDogSpan, TraceLibraryPayloadFormat +from utils.dd_types import DataDogLibrarySpan, LibraryTraceFormat from collections import defaultdict COMPONENT_EXCEPTIONS: defaultdict[str, defaultdict[str, dict]] = defaultdict( @@ -95,7 +95,7 @@ def _validate_exception_attributes(self, attributes: dict) -> None: assert isinstance(attributes[self.type_key], str) assert isinstance(attributes[self.stacktrace_key], str) - def _validate_graphql_attributes(self, attributes: dict, span: DataDogSpan) -> None: + def _validate_graphql_attributes(self, attributes: dict, span: DataDogLibrarySpan) -> None: """Validate GraphQL-specific attributes (path, locations)""" for path in attributes[self.path_key]: assert isinstance(path, str), attributes @@ -135,14 +135,14 @@ def _is_graphql_execute_span(span: dict) -> bool: return name == COMPONENT_EXCEPTIONS[lang][component]["operation_name"] @staticmethod - def _has_location(span: DataDogSpan) -> bool: + def _has_location(span: DataDogLibrarySpan) -> bool: lang = span.get("meta", {}).get("language", "") component = span.get("meta", {}).get("component", "") return COMPONENT_EXCEPTIONS[lang][component]["has_location"] @staticmethod - def _get_events(span: DataDogSpan) -> list[dict]: - if span.trace.format == TraceLibraryPayloadFormat.v10: + def _get_events(span: DataDogLibrarySpan) -> list[dict]: + if span.trace.format == LibraryTraceFormat.v10: return span["span_events"] if "events" in span["meta"]: diff --git a/tests/test_identify.py b/tests/test_identify.py index 3646b6bcfd1..f968b7f093c 100644 --- a/tests/test_identify.py +++ b/tests/test_identify.py @@ -3,10 +3,10 @@ # Copyright 2021 Datadog, Inc. from utils import weblog, interfaces, rfc, features -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan -def assert_tag_in_span_meta(span: DataDogSpan, tag: str, expected: str): +def assert_tag_in_span_meta(span: DataDogLibrarySpan, tag: str, expected: str): if tag not in span["meta"]: raise Exception(f"Can't find {tag} in span's meta") @@ -16,7 +16,7 @@ def assert_tag_in_span_meta(span: DataDogSpan, tag: str, expected: str): def validate_identify_tags(tags: dict[str, str] | list[str]): - def inner_validate(span: DataDogSpan): + def inner_validate(span: DataDogLibrarySpan): for tag in tags: if isinstance(tags, dict): assert_tag_in_span_meta(span, tag, tags[tag]) @@ -98,7 +98,7 @@ def setup_identify_tags_incoming(self): def test_identify_tags_incoming(self): """With W3C : this test expect to fail with DD_TRACE_PROPAGATION_STYLE_INJECT=W3C""" - def usr_id_not_present(span: DataDogSpan): + def usr_id_not_present(span: DataDogLibrarySpan): if "usr.id" in span["meta"]: raise Exception("usr.id must not be present in this span") return True diff --git a/tests/test_library_conf.py b/tests/test_library_conf.py index 1918d80487c..9b8a4719efc 100644 --- a/tests/test_library_conf.py +++ b/tests/test_library_conf.py @@ -3,7 +3,7 @@ # Copyright 2021 Datadog, Inc. import pytest from utils import weblog, interfaces, scenarios, features, missing_feature -from utils.dd_constants import TraceAgentPayloadFormat +from utils.dd_types import DataDogAgentSpan, AgentTraceFormat from utils._context.header_tag_vars import ( CONFIG_COLON_LEADING, CONFIG_COLON_TRAILING, @@ -402,9 +402,9 @@ def test_trace_header_tags(self): spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1 - span, span_format = spans[0] + span = spans[0] for tag in tags: - assert tag in interfaces.agent.get_span_meta(span, span_format) + assert tag in span.meta @scenarios.tracing_config_nondefault @@ -420,8 +420,8 @@ def test_trace_header_tags(self): spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1 - span, span_format = spans[0] - span_meta = interfaces.agent.get_span_meta(span, span_format) + span = spans[0] + span_meta = span.meta for key in response_headers: assert RESPONSE_PREFIX + key.lower() in span_meta assert span_meta[RESPONSE_PREFIX + key.lower()] == response_headers[key] @@ -431,22 +431,20 @@ def test_trace_header_tags(self): TRACECONTEXT_FLAGS_SET = 1 << 31 -def retrieve_span_links( - span: dict, span_format: TraceAgentPayloadFormat -) -> tuple[list[dict] | None, TraceAgentPayloadFormat]: +def retrieve_span_links(span: DataDogAgentSpan) -> list[dict] | None: """Retrieves span links from a span. Returns the format of the span links as it may differ from the trace format emitted by the agent """ if span.get("spanLinks") is not None: - return span["spanLinks"], span_format + return span["spanLinks"] - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format and span.get("links") is not None: - return span["links"], span_format + if span.trace.format == AgentTraceFormat.efficient_trace_payload_format and span.get("links") is not None: + return span["links"] - span_meta = interfaces.agent.get_span_meta(span, span_format) + span_meta = span.meta if span_meta.get("_dd.span_links") is None: - return None, span_format + return None # Convert span_links tags into msgpack v0.4 format json_links = json.loads(span_meta["_dd.span_links"]) @@ -468,7 +466,7 @@ def retrieve_span_links( else: link["flags"] = 0 links.append(link) - return links, TraceAgentPayloadFormat.legacy # Legacy format is used for span links that arrived in the meta + return links @scenarios.default @@ -491,16 +489,16 @@ def setup_single_tracecontext(self): def test_single_tracecontext(self): interfaces.library.assert_trace_exists(self.r) - traces = [(trace, trace_format) for _, trace, trace_format in interfaces.agent.get_traces(self.r)] - trace_id = interfaces.agent.get_trace_id(traces[0][0], traces[0][1]) + traces = [trace for _, trace in interfaces.agent.get_traces(self.r)] + trace_id = traces[0].trace_id_as_int spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] - assert trace_id == "1" + span = spans[0] + assert trace_id == 1 assert span.get("parentID") == "1" - span_links, _ = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is None # Test the next outbound span context @@ -528,24 +526,24 @@ def setup_multiple_tracecontexts(self): def test_multiple_tracecontexts(self): interfaces.library.assert_trace_exists(self.r) - traces = [(trace, trace_format) for _, trace, trace_format in interfaces.agent.get_traces(self.r)] + traces = [trace for _, trace in interfaces.agent.get_traces(self.r)] spans = interfaces.agent.get_spans_list(self.r) assert len(spans) == 1, "Agent received the incorrect amount of spans" assert len(traces) > 0, "Agent received the incorrect amount of traces" # Test the extracted span context - span, span_format = spans[0] - trace_id = interfaces.agent.get_trace_id(traces[0][0], traces[0][1]) - assert trace_id == "2" + span = spans[0] + trace_id = traces[0].trace_id_as_int + assert trace_id == 2, traces[0].format assert span.get("parentID") == "2" # Test the extracted span links: One span link per conflicting trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) # Assert the W3C Trace Context (conflicting trace context) span link according to the format assert trace_id_low == 8687463697196027922 # int(0x7890123456789012) assert trace_id_high == 1311768467284833366 # int(0x1234567890123456) @@ -586,20 +584,20 @@ def test_single_tracecontext(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] + span = spans[0] assert span.get("traceID") != "1" assert span.get("parentID") is None # Test the extracted span links: One span link for the incoming (Datadog trace context). # In the case that span links are generated for conflicting trace contexts, those span links # are not included in the new trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 # Assert the Datadog (restarted) span link link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) assert trace_id_low == 1 assert trace_id_high == 1229782938247303441 assert int(link["spanID"]) == 1 @@ -634,7 +632,7 @@ def test_multiple_tracecontexts(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] + span = spans[0] assert ( span.get("traceID") != "1" # Lower 64-bits of traceparent ) @@ -646,13 +644,13 @@ def test_multiple_tracecontexts(self): # Test the extracted span links: One span link for the incoming (Datadog trace context). # In the case that span links are generated for conflicting trace contexts, those span links # are not included in the new trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 # Assert the Datadog (restarted) span link link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) assert trace_id_low == 1 assert trace_id_high == 1229782938247303441 assert int(link["spanID"]) == 1 @@ -687,7 +685,7 @@ def test_multiple_tracecontexts_with_overrides(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] + span = spans[0] assert ( span.get("traceID") != "1" # Lower 64-bits of traceparent ) @@ -698,13 +696,13 @@ def test_multiple_tracecontexts_with_overrides(self): # Test the extracted span links: One span link for the incoming (Datadog trace context). # In the case that span links are generated for conflicting trace contexts, those span links # are not included in the new trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 # Assert the Datadog (restarted) span link link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) assert trace_id_low == 1 assert trace_id_high == 1229782938247303441 @@ -721,9 +719,9 @@ def test_multiple_tracecontexts_with_overrides(self): assert "key1=value1" in data["request_headers"]["baggage"] -def _get_span_link_trace_id(link: dict, span_format: TraceAgentPayloadFormat) -> tuple[int, int]: +def _get_span_link_trace_id(link: dict, span_format: AgentTraceFormat) -> tuple[int, int]: """Returns the trace ID of a span link according to its format split into high and low 64 bits""" - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: + if span_format == AgentTraceFormat.efficient_trace_payload_format: trace_id_low = int(link["traceID"], 16) & 0xFFFFFFFFFFFFFFFF trace_id_high = (int(link["traceID"], 16) >> 64) & 0xFFFFFFFFFFFFFFFF else: @@ -756,10 +754,10 @@ def test_single_tracecontext(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the local span context - span, span_format = spans[0] + span = spans[0] assert span.get("traceID") != "1" assert span.get("parentID") is None - span_links, _ = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is None # Test the next outbound span context @@ -791,7 +789,7 @@ def test_multiple_tracecontexts(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the local span context - span, span_format = spans[0] + span = spans[0] assert ( span.get("traceID") != "1" # Lower 64-bits of traceparent ) @@ -799,7 +797,7 @@ def test_multiple_tracecontexts(self): span.get("traceID") != "8687463697196027922" # Lower 64-bits of traceparent ) assert span.get("parentID") is None - span_links, _ = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is None # Test the next outbound span context @@ -836,20 +834,20 @@ def test_single_tracecontext(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] + span = spans[0] assert span.get("traceID") != "1" assert span.get("parentID") is None # Test the extracted span links: One span link for the incoming (Datadog trace context). # In the case that span links are generated for conflicting trace contexts, those span links # are not included in the new trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 # Assert the Datadog (restarted) span link link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) assert trace_id_low == 1 assert int(link["spanID"]) == 1 assert trace_id_high == 1229782938247303441 @@ -884,7 +882,7 @@ def test_multiple_tracecontexts(self): assert len(spans) == 1, "Agent received the incorrect amount of spans" # Test the extracted span context - span, span_format = spans[0] + span = spans[0] assert ( span.get("traceID") != "1" # Lower 64-bits of traceparent ) @@ -896,13 +894,13 @@ def test_multiple_tracecontexts(self): # Test the extracted span links: One span link for the incoming (Datadog trace context). # In the case that span links are generated for conflicting trace contexts, those span links # are not included in the new trace context - span_links, span_links_format = retrieve_span_links(span, span_format) + span_links = retrieve_span_links(span) assert span_links is not None, "Expected span links to be present" assert len(span_links) == 1 # Assert the Datadog (restarted) span link link = span_links[0] - trace_id_high, trace_id_low = _get_span_link_trace_id(link, span_links_format) + trace_id_high, trace_id_low = _get_span_link_trace_id(link, span.trace.format) assert trace_id_low == 1 assert trace_id_high == 1229782938247303441 assert int(link["spanID"]) == 1 diff --git a/tests/test_protobuf.py b/tests/test_protobuf.py index f687cfabc09..62a395d41f2 100644 --- a/tests/test_protobuf.py +++ b/tests/test_protobuf.py @@ -1,7 +1,7 @@ import json from utils import weblog, interfaces, features -from utils.dd_types import DataDogTrace +from utils.dd_types import DataDogLibraryTrace # this test relies on the proto file at utils/build/docker/common/message.proto @@ -15,7 +15,7 @@ def test_protobuf(self): assert self.serialization_response.status_code == 200, self.serialization_response.text assert self.deserialization_response.status_code == 200, self.deserialization_response.text - def validator(trace: DataDogTrace) -> bool: + def validator(trace: DataDogLibraryTrace) -> bool: if len(trace) == 1: span = trace[0] else: diff --git a/tests/test_sampling_rates.py b/tests/test_sampling_rates.py index e33cf6ea881..aadcb3f8785 100644 --- a/tests/test_sampling_rates.py +++ b/tests/test_sampling_rates.py @@ -11,7 +11,7 @@ from utils import weblog, interfaces, context, scenarios, features, logger from utils.dd_constants import SamplingPriority -from utils.dd_types import DataDogSpan, DataDogTrace +from utils.dd_types import DataDogLibrarySpan, DataDogLibraryTrace """Those are the constants used by the sampling algorithm in all the tracers @@ -22,7 +22,7 @@ MAX_UINT64 = 2**64 - 1 -def get_trace_request_path(root_span: DataDogSpan) -> str | None: +def get_trace_request_path(root_span: DataDogLibrarySpan) -> str | None: if root_span.get("type") != "web": return None @@ -150,7 +150,7 @@ def setup_sampling_decision(self): def test_sampling_decision(self): """Verify that traces are sampled following the sample rate""" - def validator(trace: DataDogTrace, root_span: DataDogSpan): + def validator(trace: DataDogLibraryTrace, root_span: DataDogLibrarySpan): sampling_priority = root_span["metrics"].get("_sampling_priority_v1") if sampling_priority is None: raise ValueError( diff --git a/tests/test_scrubbing.py b/tests/test_scrubbing.py index 38486fd25b0..36c58c8416c 100644 --- a/tests/test_scrubbing.py +++ b/tests/test_scrubbing.py @@ -6,7 +6,7 @@ import re from utils import context, interfaces, rfc, weblog, missing_feature, features, scenarios, logger -from utils.dd_types import DataDogTrace +from utils.dd_types import DataDogLibraryTrace def validate_no_leak(needle: str, whitelist_pattern: str | None = None) -> Callable[[dict], None]: @@ -81,7 +81,7 @@ def test_main(self): """Check that not data is leaked""" assert self.r.status_code == 200 - def validate_report(trace: DataDogTrace): + def validate_report(trace: DataDogLibraryTrace): for span in trace: if span.get("type") == "http": logger.info(f"span found: {span}") diff --git a/tests/test_semantic_conventions.py b/tests/test_semantic_conventions.py index cdba118cb0c..bdcda5ee655 100644 --- a/tests/test_semantic_conventions.py +++ b/tests/test_semantic_conventions.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse from utils import context, interfaces, features, scenarios -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan RUNTIME_LANGUAGE_MAP = { @@ -174,7 +174,7 @@ class Test_Meta: def test_meta_span_kind(self): """Validates that traces from an http framework carry a span.kind meta tag, with value server or client""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return None @@ -191,7 +191,7 @@ def validator(span: DataDogSpan): def test_meta_http_url(self): """Validates that traces from an http framework carry a http.url meta tag, formatted as a URL""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return None @@ -210,7 +210,7 @@ def validator(span: DataDogSpan): def test_meta_http_status_code(self): """Validates that traces from an http framework carry a http.status_code meta tag, formatted as a int""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return None @@ -228,7 +228,7 @@ def validator(span: DataDogSpan): def test_meta_http_method(self): """Validates that traces from an http framework carry a http.method meta tag, with a legal HTTP method""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return None @@ -262,7 +262,7 @@ def validator(span: DataDogSpan): def test_meta_language_tag(self): """Assert that all spans have required language tag.""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return @@ -283,7 +283,7 @@ def validator(span: DataDogSpan): def test_meta_component_tag(self): """Assert that all spans generated from a weblog_variant have component metadata tag matching integration name.""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("type") != "web": # do nothing if is not web related return @@ -310,7 +310,7 @@ def validator(span: DataDogSpan): def test_meta_runtime_id_tag(self): """Assert that all spans generated from a weblog_variant have runtime-id metadata tag with some value.""" - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): if span.get("parent_id") not in (0, None): # do nothing if not root span return @@ -326,7 +326,7 @@ class Test_MetaDatadogTags: """Spans carry meta tags that were set in DD_TAGS tracer environment""" def test_meta_dd_tags(self): - def validator(span: DataDogSpan): + def validator(span: DataDogLibrarySpan): assert span["meta"]["key1"] == "val1", ( f'keyTag tag in span\'s meta should be "test", not {span["meta"]["env"]}' ) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 31c7ccb62af..7c7f8e9519d 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -661,9 +661,9 @@ def test_traces_contain_install_id(self): """Assert that at least one trace carries APM onboarding info""" def validate_at_least_one_span_with_tag(tag: str): - for _, chunk, chunk_format in interfaces.agent.get_traces(): - for span in chunk.get("spans", []): - span_meta = interfaces.agent.get_span_meta(span, chunk_format) + for _, chunk in interfaces.agent.get_traces(): + for span in chunk.spans: + span_meta = span.meta if tag in span_meta: return raise Exception(f"Did not find tag {tag} in any spans") diff --git a/tests/test_v1_payloads.py b/tests/test_v1_payloads.py index 6dca2069dc7..9c894ab7272 100644 --- a/tests/test_v1_payloads.py +++ b/tests/test_v1_payloads.py @@ -1,6 +1,6 @@ from utils import weblog, interfaces, scenarios, features from utils.dd_constants import SamplingPriority, SamplingMechanism, SpanKind -from utils.dd_types import TraceLibraryPayloadFormat +from utils.dd_types import AgentTraceFormat, LibraryTraceFormat @features.efficient_trace_payload @@ -10,7 +10,7 @@ class Test_V1PayloadByDefault: def test_main(self): for data, trace in interfaces.library.get_traces(): assert data["path"] == "/v1.0/traces" - assert trace.format == TraceLibraryPayloadFormat.v10 + assert trace.format == LibraryTraceFormat.v10 @scenarios.apm_tracing_efficient_payload @@ -20,23 +20,25 @@ def setup_field_changes(self): self.r = weblog.get("/status?code=500") def test_field_changes(self): - traces = list(interfaces.library.get_traces_v1(self.r)) - agent_chunks = list(interfaces.agent.get_chunks_v1(self.r)) - assert len(traces) == 1 - _, trace = traces[0] - assert len(trace["spans"]) == 1 - assert len(trace["trace_id"]) == 34 # 32 bytes for ID and 2 for "0x" + library_traces = list(interfaces.library.get_traces(self.r)) + agent_traces = list(interfaces.agent.get_traces(self.r)) + assert len(library_traces) == 1 + _, trace = library_traces[0] + assert isinstance(trace.raw_trace, dict) + assert len(trace.raw_trace["spans"]) == 1 + assert len(trace.raw_trace["trace_id"]) == 34 # 32 bytes for ID and 2 for "0x" assert ( - trace["sampling_mechanism"] == SamplingMechanism.RULE_RATE + trace.raw_trace["sampling_mechanism"] == SamplingMechanism.RULE_RATE ) # TODO: Why is this local rule sampler for go? - assert trace["priority"] == SamplingPriority.USER_KEEP - span = trace["spans"][0] - assert span["error"], "Error field must be boolean" - assert span["env"] == "system-tests" - assert span["component"] == "net/http" - assert span["span_kind"] == SpanKind.SERVER + assert trace.raw_trace["priority"] == SamplingPriority.USER_KEEP + span = trace.spans[0] + assert span.raw_span["error"], "Error field must be boolean" + assert span.raw_span["env"] == "system-tests" + assert span.raw_span["component"] == "net/http" + assert span.raw_span["span_kind"] == SpanKind.SERVER - assert len(agent_chunks) == 1 - _, agent_chunk = agent_chunks[0] - assert len(agent_chunk["spans"]) == 1 - assert agent_chunk["traceID"] == trace["trace_id"] + assert len(agent_traces) == 1 + _, agent_trace = agent_traces[0] + assert agent_trace.format == AgentTraceFormat.efficient_trace_payload_format + assert len(agent_trace.spans) == 1 + assert agent_trace.trace_id == trace.raw_trace["trace_id"] diff --git a/utils/dd_constants.py b/utils/dd_constants.py index 5f8a230c285..987afec742d 100644 --- a/utils/dd_constants.py +++ b/utils/dd_constants.py @@ -1,4 +1,4 @@ -from enum import IntEnum, StrEnum +from enum import IntEnum # Key used in the metrics map to indicate tracer sampling priority @@ -114,17 +114,3 @@ class SpanKind(IntEnum): CLIENT = 3 PRODUCER = 4 CONSUMER = 5 - - -class TraceAgentPayloadFormat(StrEnum): - """Describe which format is used to carry trace payloads from the agent to the backend - This enum is used only in system-tests to differentiate between different agent payloads - and is not exposed directly in trace payloads. - """ - - legacy = "legacy" - """ Legacy format before agent version 7.73.0""" - - efficient_trace_payload_format = "efficient_trace_payload_format" - """ Efficient format introduced in agent version 7.73.0. Uses idxTracerPayloads field instead of tracerPayloads - RFC: https://docs.google.com/document/d/1hNS6anKYutOYW-nmR759UlKXUdT6H0mRwVt7_L70ESc/edit?usp=sharing""" diff --git a/utils/dd_types/__init__.py b/utils/dd_types/__init__.py index 7375539bd48..de1b988f7d7 100644 --- a/utils/dd_types/__init__.py +++ b/utils/dd_types/__init__.py @@ -1,3 +1,11 @@ -from ._datadog_trace import DataDogTrace, DataDogSpan, TraceLibraryPayloadFormat +from ._datadog_library_trace import DataDogLibraryTrace, DataDogLibrarySpan, LibraryTraceFormat +from ._datadog_agent_trace import DataDogAgentSpan, DataDogAgentTrace, AgentTraceFormat -__all__ = ["DataDogSpan", "DataDogTrace", "TraceLibraryPayloadFormat"] +__all__ = [ + "AgentTraceFormat", + "DataDogAgentSpan", + "DataDogAgentTrace", + "DataDogLibrarySpan", + "DataDogLibraryTrace", + "LibraryTraceFormat", +] diff --git a/utils/dd_types/_datadog_agent_trace.py b/utils/dd_types/_datadog_agent_trace.py new file mode 100644 index 00000000000..bed26169538 --- /dev/null +++ b/utils/dd_types/_datadog_agent_trace.py @@ -0,0 +1,199 @@ +from abc import ABC, abstractmethod +from enum import StrEnum +from typing import Any + + +class AgentTraceFormat(StrEnum): + """Describe which format is used to carry trace payloads from the agent to the backend""" + + legacy = "legacy" + """ Legacy format before agent version 7.73.0""" + + efficient_trace_payload_format = "efficient_trace_payload_format" + """ Efficient format introduced in agent version 7.73.0. Uses idxTracerPayloads field instead of tracerPayloads + RFC: https://docs.google.com/document/d/1hNS6anKYutOYW-nmR759UlKXUdT6H0mRwVt7_L70ESc/edit?usp=sharing""" + + +class DataDogAgentTrace(ABC): + """Wrapper around trace object reported by dd-trace libraries""" + + data: dict + """raw request and response sent to the agent""" + + format: AgentTraceFormat + + raw_trace: dict | list[dict] + """raw trace object""" + + spans: list["DataDogAgentSpan"] + + @staticmethod + def from_agent_legacy(data: dict, raw_trace: dict) -> "DataDogTraceAgentLegacy": + return DataDogTraceAgentLegacy(data, raw_trace) + + @staticmethod + def from_agent_v1(data: dict, raw_trace: dict) -> "DataDogTraceAgentV1": + return DataDogTraceAgentV1(data, raw_trace) + + @property + @abstractmethod + def trace_id(self) -> int | str: + """Returns the trace ID of a trace, as it reported by the agent withtout changing anything: + * legacy format: string, reprsenting a decimal number + * efficient format: string, representing an hexadecimal number, with 8 extra bytes the are a mystery right now + """ + + @property + @abstractmethod + def trace_id_as_int(self) -> int: + """Returns the trace ID of a trace + Returns only the lower 64 bits of the trace ID + """ + + @property + def log_filename(self) -> str: + return self.data["log_filename"] + + def __len__(self) -> int: + """Return span count""" + return len(self.spans) + + +class DataDogTraceAgentLegacy(DataDogAgentTrace): + def __init__(self, data: dict, raw_trace: dict): + self.data = data + + self.raw_trace: dict = raw_trace + + self.format = AgentTraceFormat.legacy + + self.spans = [DataDogAgentSpanLegacy(self, s) for s in self.raw_trace["spans"]] + + @property + def trace_id(self) -> str: + """The legacy format expose the trace id as a string, reprsenting a decimal number""" + return self.spans[0]["traceID"] + + @property + def trace_id_as_int(self) -> int: + return int(self.trace_id) + + +class DataDogTraceAgentV1(DataDogAgentTrace): + # spans: list["DataDogAgentSpanV10"] + + def __init__(self, data: dict, raw_trace: dict): + self.data = data + + self.raw_trace: dict = raw_trace + + self.format = AgentTraceFormat.efficient_trace_payload_format + + self.spans = [DataDogAgentSpanV10(self, s) for s in self.raw_trace["spans"]] + + @property + def trace_id(self) -> str: + return self.raw_trace["traceID"] + + @property + def trace_id_as_int(self) -> int: + return int(self.trace_id, 16) & 0xFFFFFFFFFFFFFFFF + + +class DataDogAgentSpan(ABC): + """Wrapper around trace object reported by dd-trace libraries""" + + def __init__(self, trace: DataDogAgentTrace, raw_span: dict): + self.trace = trace + + self.raw_span = raw_span + + def __contains__(self, key: str) -> bool: + return key in self.raw_span + + @abstractmethod + def get(self, key: str, default: Any = None): # noqa: ANN401 + pass + + @abstractmethod + def __getitem__(self, key: str): + pass + + @property + @abstractmethod + def meta(self) -> dict[str, Any]: + pass + + @abstractmethod + def get_span_type(self) -> str: + pass + + @abstractmethod + def get_span_name(self) -> str: + pass + + @abstractmethod + def get_span_resource(self) -> str: + pass + + @abstractmethod + def get_span_service(self) -> str: + pass + + @abstractmethod + def get_span_kind(self) -> str: + pass + + +class DataDogAgentSpanLegacy(DataDogAgentSpan): + def get(self, key: str, default: Any = None): # noqa: ANN401 + return self.raw_span.get(key, default) + + def __getitem__(self, key: str): + return self.raw_span[key] + + @property + def meta(self) -> dict[str, Any]: + return self.raw_span["meta"] + + def get_span_type(self) -> str: + return self.raw_span.get("type", "") + + def get_span_name(self) -> str: + return self.raw_span["name"] + + def get_span_resource(self) -> str: + return self.raw_span["resource"] + + def get_span_service(self) -> str: + return self.raw_span["service"] + + def get_span_kind(self) -> str: + return self.meta["span.kind"] + + +class DataDogAgentSpanV10(DataDogAgentSpan): + def get(self, key: str, default: Any = None): # noqa: ANN401 + return self.raw_span.get(key, default) + + def __getitem__(self, key: str): + return self.raw_span[key] + + @property + def meta(self) -> dict[str, Any]: + return self.raw_span["attributes"] + + def get_span_type(self) -> str: + return self.raw_span.get("typeRef", "") + + def get_span_name(self) -> str: + return self.raw_span["nameRef"] + + def get_span_resource(self) -> str: + return self.raw_span["resourceRef"] + + def get_span_service(self) -> str: + return self.raw_span["serviceRef"] + + def get_span_kind(self) -> str: + return self.raw_span["kind"] diff --git a/utils/dd_types/_datadog_trace.py b/utils/dd_types/_datadog_library_trace.py similarity index 72% rename from utils/dd_types/_datadog_trace.py rename to utils/dd_types/_datadog_library_trace.py index 4e2b5bd720e..2d327184fc4 100644 --- a/utils/dd_types/_datadog_trace.py +++ b/utils/dd_types/_datadog_library_trace.py @@ -4,7 +4,7 @@ from typing import Any -class TraceLibraryPayloadFormat(StrEnum): +class LibraryTraceFormat(StrEnum): """Describe which format is used to carry trace payloads from the library to the agent This enum is used only in system-tests to differentiate between different library payloads and is not exposed directly in trace payloads. @@ -21,26 +21,26 @@ class TraceLibraryPayloadFormat(StrEnum): RFC: https://docs.google.com/document/d/1hNS6anKYutOYW-nmR759UlKXUdT6H0mRwVt7_L70ESc/edit?usp=sharing""" -class DataDogTrace(ABC): +class DataDogLibraryTrace(ABC): """Wrapper around trace object reported by dd-trace libraries""" data: dict """raw request and response sent to the agent""" - format: TraceLibraryPayloadFormat + format: LibraryTraceFormat raw_trace: dict | list[dict] """raw trace object""" - spans: list["DataDogSpan"] + spans: list["DataDogLibrarySpan"] @staticmethod - def from_legacy(data: dict, raw_trace: list[dict]) -> "DataDogTrace": - return DataDogTraceLegacy(data, raw_trace) + def from_legacy(data: dict, raw_trace: list[dict]) -> "DataDogLibraryTrace": + return DataDogLibraryTraceLegacy(data, raw_trace) @staticmethod - def from_v1(data: dict, raw_trace: dict) -> "DataDogTracev1": - return DataDogTracev1(data, raw_trace) + def from_v1(data: dict, raw_trace: dict) -> "DataDogLibraryTracev1": + return DataDogLibraryTracev1(data, raw_trace) @property @abstractmethod @@ -63,11 +63,11 @@ def trace_id_equals(self, other: int | str) -> bool: return other == self.trace_id_as_int - def __iter__(self) -> Iterator["DataDogSpan"]: + def __iter__(self) -> Iterator["DataDogLibrarySpan"]: """Iterate over spans""" yield from self.spans - def __getitem__(self, i: int) -> "DataDogSpan": + def __getitem__(self, i: int) -> "DataDogLibrarySpan": """Get the ith spans""" return self.spans[i] @@ -76,18 +76,18 @@ def __len__(self) -> int: return len(self.spans) -class DataDogTraceLegacy(DataDogTrace): +class DataDogLibraryTraceLegacy(DataDogLibraryTrace): def __init__(self, data: dict, raw_trace: list[dict]): self.data = data self.raw_trace: list[dict] = raw_trace - self.format: TraceLibraryPayloadFormat = { - "/v0.4/traces": TraceLibraryPayloadFormat.v04, - "/v0.5/traces": TraceLibraryPayloadFormat.v05, + self.format: LibraryTraceFormat = { + "/v0.4/traces": LibraryTraceFormat.v04, + "/v0.5/traces": LibraryTraceFormat.v05, }[data["path"]] - self.spans = [DataDogSpanLegacy(self, s) for s in self.raw_trace] + self.spans = [DataDogLibrarySpanLegacy(self, s) for s in self.raw_trace] @property def trace_id(self) -> int: @@ -98,15 +98,15 @@ def trace_id_as_int(self) -> int: return self.trace_id -class DataDogTracev1(DataDogTrace): +class DataDogLibraryTracev1(DataDogLibraryTrace): def __init__(self, data: dict, raw_trace: dict): self.data = data self.raw_trace: dict = raw_trace - self.format = TraceLibraryPayloadFormat.v10 + self.format = LibraryTraceFormat.v10 - self.spans = [DataDogSpanV1(self, s) for s in self.raw_trace["spans"]] + self.spans = [DataDogLibrarySpanV1(self, s) for s in self.raw_trace["spans"]] @property def trace_id(self) -> str | int: @@ -117,10 +117,10 @@ def trace_id_as_int(self) -> int: return int(self.raw_trace["trace_id"], 16) & 0xFFFFFFFFFFFFFFFF -class DataDogSpan(ABC): +class DataDogLibrarySpan(ABC): """Wrapper around trace object reported by dd-trace libraries""" - def __init__(self, trace: DataDogTrace, raw_span: dict): + def __init__(self, trace: DataDogLibraryTrace, raw_span: dict): self.trace = trace self.raw_span = raw_span @@ -136,16 +136,26 @@ def get(self, key: str, default: Any = None): # noqa: ANN401 def __getitem__(self, key: str): pass + @property + @abstractmethod + def meta(self) -> dict[str, Any]: + pass + -class DataDogSpanLegacy(DataDogSpan): +class DataDogLibrarySpanLegacy(DataDogLibrarySpan): def get(self, key: str, default: Any = None): # noqa: ANN401 return self.raw_span.get(key, default) def __getitem__(self, key: str): return self.raw_span[key] + @property + def meta(self) -> dict[str, Any]: + assert "meta" in self.raw_span + return self.raw_span["meta"] + -class DataDogSpanV1(DataDogSpan): +class DataDogLibrarySpanV1(DataDogLibrarySpan): def __contains__(self, key: str) -> bool: if key in ("meta", "meta_struct", "metrics"): return "attributes" in self.raw_span @@ -172,3 +182,8 @@ def __getitem__(self, key: str): return self.raw_span["attributes"] return self.raw_span[key] + + @property + def meta(self) -> dict[str, Any]: + assert "attributes" in self.raw_span + return self.raw_span["attributes"] diff --git a/utils/interfaces/_agent.py b/utils/interfaces/_agent.py index 392f4b1b3b5..358da2dc9b2 100644 --- a/utils/interfaces/_agent.py +++ b/utils/interfaces/_agent.py @@ -8,7 +8,7 @@ import copy import threading -from utils.dd_constants import TraceAgentPayloadFormat +from utils.dd_types import DataDogAgentTrace, DataDogAgentSpan, AgentTraceFormat from utils.tools import get_rid_from_span from utils._logger import logger from utils.interfaces._core import ProxyBasedInterfaceValidator @@ -79,7 +79,7 @@ def get_telemetry_data(self, *, flatten_message_batches: bool = True): yield data def assert_trace_exists(self, request: HttpResponse): - for _, _, _ in self.get_traces(request=request): + for _, _ in self.get_traces(request=request): return raise ValueError(f"No trace has been found for request {request.get_rid()}") @@ -95,7 +95,7 @@ def assert_headers_presence( validator = HeadersPresenceValidator(request_headers, response_headers, check_condition) self.validate_all(validator, path_filters=path_filter, allow_no_data=True) - def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dict, dict, TraceAgentPayloadFormat]]: + def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dict, DataDogAgentTrace]]: """Attempts to fetch the traces the agent will submit to the backend. When a valid request is given, then we filter the spans to the ones sampled @@ -110,31 +110,31 @@ def get_traces(self, request: HttpResponse | None = None) -> Generator[tuple[dic for data in self.get_data(path_filters="/api/v0.2/traces"): logger.debug(f"Looking at agent data {data['log_filename']}") - if "tracerPayloads" in data["request"]["content"]: - content = data["request"]["content"]["tracerPayloads"] - - for payload in content: - for trace in payload["chunks"]: - for span in trace["spans"]: - if rid is None or get_rid_from_span(span) == rid: - logger.info(f"Found a trace in {data['log_filename']}") - yield data, trace, TraceAgentPayloadFormat.legacy - break - if "idxTracerPayloads" in data["request"]["content"]: + builder: Callable[[dict, dict], DataDogAgentTrace] + + if "tracerPayloads" in data["request"]["content"]: + builder = DataDogAgentTrace.from_agent_legacy + content: list[dict] = data["request"]["content"]["tracerPayloads"] + elif "idxTracerPayloads" in data["request"]["content"]: + builder = DataDogAgentTrace.from_agent_v1 content: list[dict] = data["request"]["content"]["idxTracerPayloads"] + else: + raise TypeError("I don't know how to build trace from this file") - for payload in content: - for trace in payload.get("chunks", []): - for span in trace["spans"]: - if rid is None or get_rid_from_span(span) == rid: - logger.info(f"Found a trace in {data['log_filename']}") - yield data, trace, TraceAgentPayloadFormat.efficient_trace_payload_format + for payload in content: + for chunk in payload.get("chunks", []): + trace = builder(data, raw_trace=chunk) + if rid is None: + yield data, trace + else: + for span in trace.spans: + if get_rid_from_span(span) == rid: + logger.debug(f"Found a span in {trace.log_filename}") + yield data, trace break - def get_spans( - self, request: HttpResponse | None = None - ) -> Generator[tuple[dict, dict, TraceAgentPayloadFormat], None, None]: + def get_spans(self, request: HttpResponse | None = None) -> Generator[tuple[dict, DataDogAgentSpan], None, None]: """Attempts to fetch the spans the agent will submit to the backend. When a valid request is given, then we filter the spans to the ones sampled @@ -147,77 +147,25 @@ def get_spans( if rid: logger.debug(f"Will try to find agent spans related to request {rid}") - for data, trace, trace_format in self.get_traces(request=request): - for span in trace["spans"]: + for data, trace in self.get_traces(request=request): + for span in trace.spans: if rid is None or get_rid_from_span(span) == rid: - yield data, span, trace_format - - @staticmethod - def get_span_meta(span: dict, span_format: TraceAgentPayloadFormat) -> dict[str, str]: - """Returns the meta dictionary of a span according to its format""" - if span_format == TraceAgentPayloadFormat.legacy: - return span["meta"] - - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - # in the new format, metrics and meta are joined in attributes - return span["attributes"] - - raise ValueError(f"Unknown span format: {span_format}") + yield data, span @staticmethod - def set_span_attrs(span: dict, span_format: TraceAgentPayloadFormat, meta: dict[str, str]) -> None: - """Overwrites the span attributes of a span according to its format. - For legacy spans this means only the meta dictionary, - for efficient spans this means the entire attributes dictionary. - """ - if span_format == TraceAgentPayloadFormat.legacy: - span["meta"] = meta - - elif span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - span["attributes"] = meta - else: - raise ValueError(f"Unknown span format: {span_format}") - - @staticmethod - def get_span_metrics(span: dict, span_format: TraceAgentPayloadFormat) -> dict[str, str]: + def get_span_metrics(span: DataDogAgentSpan) -> dict[str, str]: """Returns the metrics dictionary of a span according to its format""" - if span_format == TraceAgentPayloadFormat.legacy: + if span.trace.format == AgentTraceFormat.legacy: return span["metrics"] - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: + if span.trace.format == AgentTraceFormat.efficient_trace_payload_format: # in the new format, metrics and meta are joined in attributes return span["attributes"] - raise ValueError(f"Unknown span format: {span_format}") - - def get_chunks_v1(self, request: HttpResponse | None = None): - """Attempts to fetch the v1 trace chunks the agent will submit to the backend. - - When a valid request is given, then we filter the chunks to the ones sampled - during that request's execution, and only return those. - """ - - rid = request.get_rid() if request else None - if rid: - logger.debug(f"Will try to find agent spans related to request {rid}") - - for data in self.get_data(path_filters="/api/v0.2/traces"): - logger.debug(f"Looking at agent data {data['log_filename']}") - if "idxTracerPayloads" not in data["request"]["content"]: - continue - content = data["request"]["content"]["idxTracerPayloads"] - - for payload in content: - logger.debug(f"Looking at agent payload {payload}") - for chunk in payload.get("chunks", []): - for span in chunk["spans"]: - if rid is None or get_rid_from_span(span) == rid: - logger.debug(f"Found a span in {data['log_filename']}") - yield data, chunk - break + raise ValueError(f"Unknown span format: {span.trace.format}") - def get_spans_list(self, request: HttpResponse | None = None) -> list[tuple[dict, TraceAgentPayloadFormat]]: - return [(span, span_format) for _, span, span_format in self.get_spans(request)] + def get_spans_list(self, request: HttpResponse | None = None) -> list[DataDogAgentSpan]: + return [span for _, span in self.get_spans(request)] def get_metrics(self): """Attempts to fetch the metrics the agent will submit to the backend.""" @@ -267,54 +215,3 @@ def get_stats(self, resource: str = ""): for client_grouped_stat in client_stats_buckets["Stats"]: if resource == "" or client_grouped_stat["Resource"] == resource: yield client_grouped_stat - - @staticmethod - def get_trace_id(chunk: dict, chunk_format: TraceAgentPayloadFormat) -> str: - """Returns the trace ID of a chunk according to its format - Returns only the lower 64 bits of the trace ID - """ - if chunk_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return str(int(chunk["traceID"], 16) & 0xFFFFFFFFFFFFFFFF) - if chunk_format == TraceAgentPayloadFormat.legacy: - return chunk["spans"][0]["traceID"] - raise ValueError(f"Unknown chunk format: {chunk_format}") - - @staticmethod - def get_span_type(span: dict, span_format: TraceAgentPayloadFormat) -> str: - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return span.get("typeRef", "") - if span_format == TraceAgentPayloadFormat.legacy: - return span.get("type", "") - raise ValueError(f"Unknown span format: {span_format}") - - @staticmethod - def get_span_name(span: dict, span_format: TraceAgentPayloadFormat) -> str: - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return span["nameRef"] - if span_format == TraceAgentPayloadFormat.legacy: - return span["name"] - raise ValueError(f"Unknown span format: {span_format}") - - @staticmethod - def get_span_resource(span: dict, span_format: TraceAgentPayloadFormat) -> str: - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return span["resourceRef"] - if span_format == TraceAgentPayloadFormat.legacy: - return span["resource"] - raise ValueError(f"Unknown span format: {span_format}") - - @staticmethod - def get_span_service(span: dict, span_format: TraceAgentPayloadFormat) -> str: - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return span["serviceRef"] - if span_format == TraceAgentPayloadFormat.legacy: - return span["service"] - raise ValueError(f"Unknown span format: {span_format}") - - @staticmethod - def get_span_kind(span: dict, span_format: TraceAgentPayloadFormat) -> str: - if span_format == TraceAgentPayloadFormat.efficient_trace_payload_format: - return span["kind"] - if span_format == TraceAgentPayloadFormat.legacy: - return span["meta"]["span.kind"] - raise ValueError(f"Unknown span format: {span_format}") diff --git a/utils/interfaces/_library/appsec.py b/utils/interfaces/_library/appsec.py index ad50ecc0d8a..9074d8f6c9e 100644 --- a/utils/interfaces/_library/appsec.py +++ b/utils/interfaces/_library/appsec.py @@ -8,7 +8,7 @@ from collections.abc import Callable from utils.interfaces._library.appsec_data import rule_id_to_type from utils._logger import logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan class _WafAttack: @@ -70,7 +70,7 @@ def _get_parameters(event: dict) -> list: return result - def validate(self, span: DataDogSpan, appsec_data: dict): + def validate(self, span: DataDogLibrarySpan, appsec_data: dict): if "triggers" not in appsec_data: logger.error("triggers is not in appsec_data") @@ -173,7 +173,7 @@ def validate_legacy(self, event: dict): return True - def validate(self, span: DataDogSpan, appsec_data: dict): # noqa: ARG002 + def validate(self, span: DataDogLibrarySpan, appsec_data: dict): # noqa: ARG002 headers = [n.lower() for n in span["meta"] if n.startswith("http.request.headers.")] assert f"http.request.headers.{self.header_name}" in headers, f"header {self.header_name} not reported" diff --git a/utils/interfaces/_library/core.py b/utils/interfaces/_library/core.py index c1f61106769..23bbbdb33eb 100644 --- a/utils/interfaces/_library/core.py +++ b/utils/interfaces/_library/core.py @@ -11,7 +11,7 @@ from utils.tools import get_rid_from_user_agent, get_rid_from_span from utils._logger import logger from utils.dd_constants import RemoteConfigApplyState, Capabilities -from utils.dd_types import DataDogSpan, DataDogTrace +from utils.dd_types import DataDogLibrarySpan, DataDogLibraryTrace from utils.interfaces._core import ProxyBasedInterfaceValidator from utils.interfaces._library.appsec import _WafAttack, _ReportedHeader from utils.interfaces._library.miscs import _SpanTagValidator @@ -62,7 +62,7 @@ def wait_function(data: dict): ############################################################ def get_traces( self, request: HttpResponse | GrpcResponse | None = None - ) -> Generator[tuple[dict, DataDogTrace], None, None]: + ) -> Generator[tuple[dict, DataDogLibraryTrace], None, None]: rid: str | None = None if request: @@ -82,13 +82,13 @@ def get_traces( for trace in content: if rid is None: trace_found = True - yield data, DataDogTrace.from_legacy(data, trace) + yield data, DataDogLibraryTrace.from_legacy(data, trace) else: for span in trace: if rid == get_rid_from_span(span): logger.debug(f"Found a trace in {data['log_filename']}") trace_found = True - yield data, DataDogTrace.from_legacy(data, trace) + yield data, DataDogLibraryTrace.from_legacy(data, trace) break elif data["path"] == "/v1.0/traces": @@ -98,13 +98,13 @@ def get_traces( for trace in content.get("chunks"): if rid is None: trace_found = True - yield data, DataDogTrace.from_v1(data, trace) + yield data, DataDogLibraryTrace.from_v1(data, trace) else: for span in trace.get("spans"): if rid == get_rid_from_span(span): logger.debug(f"Found a trace in {data['log_filename']}") trace_found = True - yield data, DataDogTrace.from_v1(data, trace) + yield data, DataDogLibraryTrace.from_v1(data, trace) break else: @@ -113,40 +113,6 @@ def get_traces( if not trace_found: logger.warning("No trace found") - def get_traces_v1(self, request: HttpResponse | GrpcResponse | None = None): - rid: str | None = None - - if request: - rid = request.get_rid() - logger.debug(f"Try to find traces related to request {rid}") - if isinstance(request, HttpResponse) and request.status_code is None: - logger.warning("HTTP app failed to respond, it will very probably fail") - - trace_found = False - for data in self.get_data(path_filters="/v1.0/traces"): - traces = data["request"]["content"] - if not traces: # may be none - continue - - if not traces.get("chunks"): - continue - - for trace in traces.get("chunks"): - if rid is None: - trace_found = True - yield data, trace - else: - for span in trace.get("spans"): - logger.debug("GOT SPAN", span) - if rid == get_rid_from_span(span): - logger.debug(f"Found a trace in {data['log_filename']}") - trace_found = True - yield data, trace - break - - if not trace_found: - logger.warning("No trace found") - def get_spans(self, request: HttpResponse | None = None, *, full_trace: bool = False): """Iterate over all spans reported by the tracer to the agent. @@ -167,12 +133,12 @@ def get_spans(self, request: HttpResponse | None = None, *, full_trace: bool = F def get_root_spans( self, request: HttpResponse | None = None - ) -> Generator[tuple[DataDogTrace, DataDogSpan], None, None]: + ) -> Generator[tuple[DataDogLibraryTrace, DataDogLibrarySpan], None, None]: for _, trace, span in self.get_spans(request=request): if span.get("parent_id") in (0, None): yield trace, span - def get_root_span(self, request: HttpResponse) -> DataDogSpan: + def get_root_span(self, request: HttpResponse) -> DataDogLibrarySpan: """Get the root span associated with a given request. This function will fail if a request is not given, if there is no root span, or if there is more than one root span. For special cases, use get_root_spans. @@ -298,7 +264,7 @@ def validator_skip_onboarding_event(data: dict) -> None: def validate_one_appsec( self, request: HttpResponse | None = None, - validator: Callable[[DataDogSpan, dict], bool] | None = None, + validator: Callable[[DataDogLibrarySpan, dict], bool] | None = None, *, legacy_validator: Callable | None = None, full_trace: bool = False, @@ -324,7 +290,7 @@ def validate_one_appsec( def validate_all_appsec( self, - validator: Callable[[DataDogSpan, dict], None], + validator: Callable[[DataDogLibrarySpan, dict], None], request: HttpResponse | None = None, *, allow_no_data: bool = False, @@ -410,7 +376,7 @@ def assert_waf_attack( key_path: str | list[str] | None = None, *, full_trace: bool = False, - span_validator: Callable[[DataDogSpan, dict], None] | None = None, + span_validator: Callable[[DataDogLibrarySpan, dict], None] | None = None, ): """Asserts the WAF detected an attack on the provided request. @@ -444,7 +410,7 @@ def add_appsec_reported_header(self, request: HttpResponse, header_name: str): def validate_all_traces(self, validator: Callable[[dict], None], *, allow_no_trace: bool = False): self.validate_all(validator=validator, allow_no_data=allow_no_trace, path_filters=r"/v0\.[1-9]+/traces") - def validate_one_trace(self, request: HttpResponse, validator: Callable[[DataDogTrace], bool]): + def validate_one_trace(self, request: HttpResponse, validator: Callable[[DataDogLibraryTrace], bool]): """Will call validator() on all traces trigerred by request. validator() returns a boolean : * True : the payload satisfies the condition, validate_one returns in success * False : the payload is ignored @@ -468,7 +434,7 @@ def validate_one_span( self, request: HttpResponse | None = None, *, - validator: Callable[[DataDogSpan], bool], + validator: Callable[[DataDogLibrarySpan], bool], full_trace: bool = False, ): """Will call validator() on all spans (eventually filtered on span trigerred by request). @@ -495,7 +461,7 @@ def validate_all_spans( self, request: HttpResponse | None = None, *, - validator: Callable[[DataDogSpan], None], + validator: Callable[[DataDogLibrarySpan], None], full_trace: bool = False, allow_no_data: bool = False, ): @@ -629,7 +595,7 @@ def assert_rc_targets_version_states(self, targets_version: int, config_states: assert found, f"Nothing has been found for targets_version {targets_version}" def assert_rasp_attack(self, request: HttpResponse, rule: str, parameters: dict | None = None): - def validator(_: DataDogSpan, appsec_data: dict): + def validator(_: DataDogLibrarySpan, appsec_data: dict): assert "triggers" in appsec_data, "'triggers' not found in '_dd.appsec.json'" triggers = appsec_data["triggers"] diff --git a/utils/tools.py b/utils/tools.py index deaf232f259..7c41b26ba7c 100644 --- a/utils/tools.py +++ b/utils/tools.py @@ -6,7 +6,7 @@ import os import re from utils._logger import logger as _logger -from utils.dd_types import DataDogSpan +from utils.dd_types import DataDogLibrarySpan, DataDogAgentSpan class ShColors(StrEnum): @@ -57,7 +57,7 @@ def e(message: str) -> str: return f"{ShColors.RED}{message}{ShColors.ENDC}" -def get_rid_from_span(span: DataDogSpan) -> str | None: +def get_rid_from_span(span: DataDogLibrarySpan | DataDogAgentSpan) -> str | None: meta = span.get("meta", {}) metrics = span.get("metrics", {})