diff --git a/aws/logs_monitoring/README.md b/aws/logs_monitoring/README.md index d2701cc23..f27ec2d6f 100644 --- a/aws/logs_monitoring/README.md +++ b/aws/logs_monitoring/README.md @@ -784,9 +784,6 @@ To test different patterns against your logs, turn on [debug logs](#troubleshoot `DD_FETCH_LOG_GROUP_TAGS` : [DEPRECATED, use DD_ENRICH_CLOUDWATCH_TAGS] Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics, and traces. If set to true, permission `logs:ListTagsForResource` will be automatically added to the Lambda execution IAM role. -`DD_FETCH_STEP_FUNCTIONS_TAGS` -: Let the Forwarder fetch Step Functions tags using GetResources API calls and apply them to logs and traces (if Step Functions tracing is enabled). If set to true, permission `tag:GetResources` will be automatically added to the Lambda execution IAM role. - `DD_STEP_FUNCTION_TRACE_ENABLED` : Set to true to enable tracing for all Step Functions. diff --git a/aws/logs_monitoring/caching/cache_layer.py b/aws/logs_monitoring/caching/cache_layer.py index eef6a535a..a63b15c35 100644 --- a/aws/logs_monitoring/caching/cache_layer.py +++ b/aws/logs_monitoring/caching/cache_layer.py @@ -1,5 +1,4 @@ from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache -from caching.step_functions_cache import StepFunctionsTagsCache from caching.s3_tags_cache import S3TagsCache from caching.lambda_cache import LambdaTagsCache @@ -8,7 +7,6 @@ class CacheLayer: def __init__(self, prefix): self._cloudwatch_log_group_cache = CloudwatchLogGroupTagsCache(prefix) self._s3_tags_cache = S3TagsCache(prefix) - self._step_functions_cache = StepFunctionsTagsCache(prefix) self._lambda_cache = LambdaTagsCache(prefix) def get_cloudwatch_log_group_tags_cache(self): @@ -17,8 +15,5 @@ def get_cloudwatch_log_group_tags_cache(self): def get_s3_tags_cache(self): return self._s3_tags_cache - def get_step_functions_tags_cache(self): - return self._step_functions_cache - def get_lambda_tags_cache(self): return self._lambda_cache diff --git a/aws/logs_monitoring/caching/step_functions_cache.py b/aws/logs_monitoring/caching/step_functions_cache.py deleted file mode 100644 index 0a240ed7a..000000000 --- a/aws/logs_monitoring/caching/step_functions_cache.py +++ /dev/null @@ -1,145 +0,0 @@ -from botocore.exceptions import ClientError - -from caching.base_tags_cache import BaseTagsCache -from caching.common import ( - parse_get_resources_response_for_tags_by_arn, - sanitize_aws_tag_string, -) -from settings import ( - DD_S3_STEP_FUNCTIONS_CACHE_FILENAME, - DD_S3_STEP_FUNCTIONS_CACHE_LOCK_FILENAME, - GET_RESOURCES_STEP_FUNCTIONS_FILTER, - get_fetch_step_functions_tags, -) -from telemetry import send_forwarder_internal_metrics - - -class StepFunctionsTagsCache(BaseTagsCache): - def __init__(self, prefix): - super().__init__( - prefix, - DD_S3_STEP_FUNCTIONS_CACHE_FILENAME, - DD_S3_STEP_FUNCTIONS_CACHE_LOCK_FILENAME, - ) - - def should_fetch_tags(self): - return get_fetch_step_functions_tags() - - def build_tags_cache(self): - """Makes API calls to GetResources to get the live tags of the account's Step Functions - Returns an empty dict instead of fetching custom tags if the tag fetch env variable is not - set to true. - Returns: - tags_by_arn_cache (dict): each Lambda's tags in a dict keyed by ARN - """ - tags_fetch_success = False - tags_by_arn_cache = {} - get_resources_paginator = self.get_resources_paginator() - - try: - for page in get_resources_paginator.paginate( - ResourceTypeFilters=[GET_RESOURCES_STEP_FUNCTIONS_FILTER], - ResourcesPerPage=100, - ): - send_forwarder_internal_metrics( - "step_functions_get_resources_api_calls" - ) - page_tags_by_arn = parse_get_resources_response_for_tags_by_arn(page) - tags_by_arn_cache.update(page_tags_by_arn) - tags_fetch_success = True - - except ClientError as e: - self.logger.error( - "Encountered a ClientError when trying to fetch tags. You may need to give " - f"this Lambda's role the 'tag:GetResources' permission: {e}" - ) - additional_tags = [ - f"http_status_code:{e.response['ResponseMetadata']['HTTPStatusCode']}" - ] - send_forwarder_internal_metrics( - "client_error", additional_tags=additional_tags - ) - - self.logger.debug( - "All Step Functions tags refreshed: {}".format(tags_by_arn_cache) - ) - - return tags_fetch_success, tags_by_arn_cache - - def get(self, state_machine_arn): - """Get the tags for the Step Functions from the cache - - Will re-fetch the tags if they are out of date, or a log group is encountered - which isn't in the tag list - - Args: - state_machine_arn (str): the key we're getting tags from the cache for - - Returns: - state_machine_tags (List[str]): the list of "key:value" Datadog tag strings - """ - if self._is_expired(): - send_forwarder_internal_metrics("local_step_functions_tags_cache_expired") - self.logger.debug( # noqa: F821 - "Local cache expired for Step Functions tags. Fetching cache from S3" - ) - self._refresh() - - state_machine_tags = self.tags_by_id.get(state_machine_arn, None) - if state_machine_tags is None: - # If the custom tag fetch env var is not set to true do not fetch - if not self.should_fetch_tags(): - self.logger.debug( - "Not fetching custom tags because the env variable DD_FETCH_STEP_FUNCTIONS_TAGS" - " is not set to true" - ) - return [] - state_machine_tags = self._get_state_machine_tags(state_machine_arn) or [] - self.tags_by_id[state_machine_arn] = state_machine_tags - - return state_machine_tags - - def _get_state_machine_tags(self, state_machine_arn: str): - """Return a list of tags of a state machine in dd format (max 200 chars) - - Example response from get source api: - { - "ResourceTagMappingList": [ - { - "ResourceARN": "arn:aws:states:us-east-1:1234567890:stateMachine:example-machine", - "Tags": [ - { - "Key": "ENV", - "Value": "staging" - } - ] - } - ] - } - - Args: - state_machine_arn (str): the key we're getting tags from the cache for - Returns: - state_machine_arn (List[str]): e.g. ["k1:v1", "k2:v2"] - """ - response = None - formatted_tags = [] - - try: - send_forwarder_internal_metrics("get_state_machine_tags") - response = self.resource_tagging_client.get_resources( - ResourceARNList=[state_machine_arn] - ) - except Exception as e: - self.logger.error(f"Failed to get Step Functions tags due to {e}") - - if response and len(response.get("ResourceTagMappingList", {})) > 0: - resource_dict = response.get("ResourceTagMappingList")[0] - for a_tag in resource_dict.get("Tags", []): - key = sanitize_aws_tag_string(a_tag["Key"], remove_colons=True) - value = sanitize_aws_tag_string( - a_tag.get("Value"), remove_leading_digits=False - ) - formatted_tags.append(f"{key}:{value}"[:200]) # same logic as lambda - - return formatted_tags diff --git a/aws/logs_monitoring/customized_log_group.py b/aws/logs_monitoring/customized_log_group.py index 475d66e3c..ad63a952a 100644 --- a/aws/logs_monitoring/customized_log_group.py +++ b/aws/logs_monitoring/customized_log_group.py @@ -26,11 +26,6 @@ def is_lambda_customized_log_group(logstream_name): ) -# For both default and customzied Step Functions log groups, the log_stream starts with "states/" -def is_step_functions_log_group(logstream_name): - return logstream_name.startswith("states/") - - def get_lambda_function_name_from_logstream_name(logstream_name): try: # Not match the pattern for customized Lambda log group diff --git a/aws/logs_monitoring/logs/datadog_http_client.py b/aws/logs_monitoring/logs/datadog_http_client.py index a72b42cd6..6880540b8 100644 --- a/aws/logs_monitoring/logs/datadog_http_client.py +++ b/aws/logs_monitoring/logs/datadog_http_client.py @@ -56,6 +56,9 @@ class DatadogHTTPClient(object): if storage_tag != "": _HEADERS["DD-STORAGE-TAG"] = storage_tag + if os.environ.get("DD_STEP_FUNCTIONS_TRACE_ENABLED", "false").lower() == "true": + _HEADERS["DD-STEP-FUNCTIONS-TRACE-ENABLED"] = "true" + def __init__( self, host, port, no_ssl, skip_ssl_validation, api_key, scrubber, timeout=10 ): diff --git a/aws/logs_monitoring/settings.py b/aws/logs_monitoring/settings.py index 5f3695412..46f1d5c6d 100644 --- a/aws/logs_monitoring/settings.py +++ b/aws/logs_monitoring/settings.py @@ -294,10 +294,6 @@ def is_api_key_valid(): "DD_FETCH_LAMBDA_TAGS", default="false", boolean=True ) -DD_FETCH_STEP_FUNCTIONS_TAGS = get_env_var( - "DD_FETCH_STEP_FUNCTIONS_TAGS", default="false", boolean=True -) - DD_ENRICH_S3_TAGS = get_env_var("DD_ENRICH_S3_TAGS", default="true", boolean=True) @@ -328,10 +324,6 @@ def get_fetch_lambda_tags(): return DD_FETCH_LAMBDA_TAGS -def get_fetch_step_functions_tags(): - return DD_FETCH_STEP_FUNCTIONS_TAGS - - def get_enrich_s3_tags(): return DD_ENRICH_S3_TAGS @@ -367,9 +359,6 @@ def get_enrich_cloudwatch_tags(): DD_S3_LAMBDA_CACHE_FILENAME = "lambda.json" DD_S3_LAMBDA_CACHE_LOCK_FILENAME = "lambda.lock" -DD_S3_STEP_FUNCTIONS_CACHE_FILENAME = "step-functions-cache.json" -DD_S3_STEP_FUNCTIONS_CACHE_LOCK_FILENAME = "step-functions-cache.lock" - DD_S3_TAGS_CACHE_FILENAME = "s3.json" DD_S3_TAGS_CACHE_LOCK_FILENAME = "s3.lock" @@ -378,7 +367,6 @@ def get_enrich_cloudwatch_tags(): DD_TAGS_CACHE_TTL_SECONDS = int(get_env_var("DD_TAGS_CACHE_TTL_SECONDS", default=300)) DD_S3_CACHE_LOCK_TTL_SECONDS = 60 GET_RESOURCES_LAMBDA_FILTER = "lambda" -GET_RESOURCES_STEP_FUNCTIONS_FILTER = "states" GET_RESOURCES_S3_FILTER = "s3:bucket" diff --git a/aws/logs_monitoring/steps/enums.py b/aws/logs_monitoring/steps/enums.py index 5e244afc6..31e7c18bc 100644 --- a/aws/logs_monitoring/steps/enums.py +++ b/aws/logs_monitoring/steps/enums.py @@ -9,7 +9,6 @@ class AwsEventSource(Enum): LAMBDA = "lambda" S3 = "s3" SNS = "sns" - STEPFUNCTION = "stepfunction" WAF = "waf" def __str__(self): diff --git a/aws/logs_monitoring/steps/handlers/awslogs_handler.py b/aws/logs_monitoring/steps/handlers/awslogs_handler.py index 48b5a707f..9de6e8532 100644 --- a/aws/logs_monitoring/steps/handlers/awslogs_handler.py +++ b/aws/logs_monitoring/steps/handlers/awslogs_handler.py @@ -3,13 +3,11 @@ import json import logging import os -import re from io import BufferedReader, BytesIO from customized_log_group import ( get_lambda_function_name_from_logstream_name, is_lambda_customized_log_group, - is_step_functions_log_group, ) from settings import DD_CUSTOM_TAGS, DD_HOST, DD_SOURCE from steps.common import ( @@ -98,9 +96,6 @@ def set_source(self, event, metadata, aws_attributes): # Need to place the handling of customized log group at the bottom so that it can correct the source for some edge cases if is_lambda_customized_log_group(log_stream): metadata[DD_SOURCE] = str(AwsEventSource.LAMBDA) - # Regardless of whether the log group is customized, the corresponding log stream starts with 'states/'." - if is_step_functions_log_group(log_stream): - metadata[DD_SOURCE] = str(AwsEventSource.STEPFUNCTION) def add_cloudwatch_tags_from_cache(self, metadata, aws_attributes): log_group_arn = aws_attributes.get_log_group_arn() @@ -128,51 +123,6 @@ def set_host(self, metadata, aws_attributes): match metadata_source: case AwsEventSource.CLOUDWATCH: metadata[DD_HOST] = log_group - case AwsEventSource.STEPFUNCTION: - self.handle_step_function_source(metadata, aws_attributes) - - def handle_verified_access_source(self, metadata, aws_attributes): - try: - message = json.loads(aws_attributes.get_log_events()[0].get("message")) - metadata[DD_HOST] = message.get("http_request").get("url").get("hostname") - except Exception as e: - logger.debug("Unable to set verified-access log host: %s" % e) - - def handle_step_function_source(self, metadata, aws_attributes): - state_machine_arn = self.get_state_machine_arn(aws_attributes) - if not state_machine_arn: - return - - metadata[DD_HOST] = state_machine_arn - formatted_stepfunctions_tags = ( - self.cache_layer.get_step_functions_tags_cache().get(state_machine_arn) - ) - if len(formatted_stepfunctions_tags) > 0: - metadata[DD_CUSTOM_TAGS] = ( - ",".join(formatted_stepfunctions_tags) - if not metadata[DD_CUSTOM_TAGS] - else metadata[DD_CUSTOM_TAGS] - + "," - + ",".join(formatted_stepfunctions_tags) - ) - - if os.environ.get("DD_STEP_FUNCTIONS_TRACE_ENABLED", "false").lower() == "true": - metadata[DD_CUSTOM_TAGS] = ",".join( - [metadata.get(DD_CUSTOM_TAGS, [])] - + ["dd_step_functions_trace_enabled:true"] - ) - - def get_state_machine_arn(self, aws_attributes): - try: - message = json.loads(aws_attributes.get_log_events()[0].get("message")) - if message.get("execution_arn") is not None: - execution_arn = message["execution_arn"] - arn_tokens = re.split(r"[:/\\]", execution_arn) - arn_tokens[5] = "stateMachine" - return ":".join(arn_tokens[:7]) - except Exception as e: - logger.debug("Unable to get state_machine_arn: %s" % e) - return "" # Lambda logs can be from either default or customized log group def process_lambda_logs(self, metadata, aws_attributes): diff --git a/aws/logs_monitoring/template.yaml b/aws/logs_monitoring/template.yaml index b03fe0edc..9969fe98e 100644 --- a/aws/logs_monitoring/template.yaml +++ b/aws/logs_monitoring/template.yaml @@ -107,13 +107,6 @@ Parameters: - true - false Description: (DEPRECATED in favor of DdEnrichCloudwatchTags) Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics and traces. If set to true, permission logs:ListTagsLogGroup will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.list_tags_log_group_api_call metric for each API call made. - DdFetchStepFunctionsTags: - Type: String - Default: true - AllowedValues: - - true - - false - Description: Let the forwarder fetch Step Functions tags using GetResources API calls and apply them to logs, metrics and traces. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made. DdFetchS3Tags: Type: String Default: false @@ -366,7 +359,6 @@ Conditions: SetDdSkipSslValidation: !Equals [!Ref DdSkipSslValidation, true] SetDdFetchLambdaTags: !Equals [!Ref DdFetchLambdaTags, true] SetDdFetchLogGroupTags: !Equals [!Ref DdFetchLogGroupTags, true] - SetDdFetchStepFunctionsTags: !Equals [!Ref DdFetchStepFunctionsTags, true] CreateS3Bucket: !And - !Or - !Equals [!Ref DdFetchLogGroupTags, true] @@ -402,7 +394,6 @@ Conditions: - !Equals [!Ref DdForwarderBucketsAccessLogsTarget, ""] ShouldDdFetchTags: !Or - !Equals [!Ref DdFetchLambdaTags, true] - - !Equals [!Ref DdFetchStepFunctionsTags, true] - !Equals [!Ref DdFetchS3Tags, true] SetForwarderBucket: !Or - !Condition CreateS3Bucket @@ -523,10 +514,6 @@ Resources: - SetDdFetchLogGroupTags - !Ref DdFetchLogGroupTags - !Ref AWS::NoValue - DD_FETCH_STEP_FUNCTIONS_TAGS: !If - - SetDdFetchStepFunctionsTags - - !Ref DdFetchStepFunctionsTags - - !Ref AWS::NoValue DD_NO_SSL: !If - SetDdNoSsl - !Ref DdNoSsl @@ -1166,7 +1153,6 @@ Metadata: - DdEnrichCloudwatchTags - DdFetchLambdaTags - DdFetchLogGroupTags - - DdFetchStepFunctionsTags - DdFetchS3Tags - DdStepFunctionsTraceEnabled - DdEnhancedMetrics diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json deleted file mode 100644 index 9ec7ce4de..000000000 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.approved.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "aws": { - "awslogs": { - "logGroup": "test/logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", - "owner": "425362996713" - }, - "invoked_function_arn": "invoked_function_arn" - }, - "ddsource": "stepfunction", - "ddsourcecategory": "aws", - "ddtags": "forwardername:function_name,forwarder_version:,test_tag_key:test_tag_value,dd_step_functions_trace_enabled:true", - "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction2", - "id": "37199773595581154154810589279545129148442535997644275712", - "message": "{\"id\": \"1\",\"type\": \"ExecutionStarted\",\"details\": {\"input\": \"{}\",\"inputDetails\": {\"truncated\": \"false\"},\"roleArn\": \"arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt\"},\"previous_event_id\": \"0\",\"event_timestamp\": \"1716992192441\",\"execution_arn\": \"arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa\",\"redrive_count\": \"0\"}", - "service": "stepfunction", - "timestamp": 1668095539607 - } -] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json deleted file mode 100644 index 3e65daa41..000000000 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_customized_log_group.metadata.approved.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "ddsource": "stepfunction", - "ddtags": "env:dev,test_tag_key:test_tag_value,dd_step_functions_trace_enabled:true", - "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction2", - "service": "stepfunction" -} diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json deleted file mode 100644 index 7669d5859..000000000 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.approved.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "aws": { - "awslogs": { - "logGroup": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", - "logStream": "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9", - "owner": "425362996713" - }, - "invoked_function_arn": "invoked_function_arn" - }, - "ddsource": "stepfunction", - "ddsourcecategory": "aws", - "ddtags": "forwardername:function_name,forwarder_version:,test_tag_key:test_tag_value,service:customservice,dd_step_functions_trace_enabled:true", - "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", - "id": "37199773595581154154810589279545129148442535997644275712", - "message": "{\"id\": \"1\",\"type\": \"ExecutionStarted\",\"details\": {\"input\": \"{}\",\"inputDetails\": {\"truncated\": \"false\"},\"roleArn\": \"arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt\"},\"previous_event_id\": \"0\",\"event_timestamp\": \"1716992192441\",\"execution_arn\": \"arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa\",\"redrive_count\": \"0\"}", - "service": "customservice", - "timestamp": 1668095539607 - } -] diff --git a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json b/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json deleted file mode 100644 index 0310f0841..000000000 --- a/aws/logs_monitoring/tests/approved_files/TestAWSLogsHandler.test_awslogs_handler_step_functions_tags_added_properly.metadata.approved.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "ddsource": "stepfunction", - "ddtags": "env:dev,test_tag_key:test_tag_value,dd_step_functions_trace_enabled:true", - "host": "arn:aws:states:us-east-1:12345678910:stateMachine:StepFunction1", - "service": "stepfunction" -} diff --git a/aws/logs_monitoring/tests/test_awslogs_handler.py b/aws/logs_monitoring/tests/test_awslogs_handler.py index 795673dd5..7c7b1c7a7 100644 --- a/aws/logs_monitoring/tests/test_awslogs_handler.py +++ b/aws/logs_monitoring/tests/test_awslogs_handler.py @@ -142,132 +142,6 @@ def test_awslogs_handler_rds_postgresql(self, mock_cache_init): options=Options().with_scrubber(self.scrubber), ) - @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") - @patch("caching.cloudwatch_log_group_cache.send_forwarder_internal_metrics") - @patch.dict( - "os.environ", - { - "DD_STEP_FUNCTIONS_TRACE_ENABLED": "true", - "DD_FETCH_STEP_FUNCTIONS_TAGS": "true", - }, - ) - def test_awslogs_handler_step_functions_tags_added_properly( - self, - mock_forward_metrics, - mock_cache_init, - ): - event = { - "awslogs": { - "data": base64.b64encode( - gzip.compress( - bytes( - json.dumps( - { - "messageType": "DATA_MESSAGE", - "owner": "425362996713", - "logGroup": ( - "/aws/vendedlogs/states/logs-to-traces-sequential-Logs" - ), - "logStream": ( - "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9" - ), - "subscriptionFilters": ["testFilter"], - "logEvents": [ - { - "id": ( - "37199773595581154154810589279545129148442535997644275712" - ), - "timestamp": 1668095539607, - "message": ( - '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction1:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}' - ), - } - ], - } - ), - "utf-8", - ) - ) - ) - } - } - mock_forward_metrics.side_effect = MagicMock() - mock_cache_init.return_value = None - cache_layer = CacheLayer("") - cache_layer._step_functions_cache.get = MagicMock( - return_value=["test_tag_key:test_tag_value", "service:customservice"] - ) - cache_layer._cloudwatch_log_group_cache.get = MagicMock() - - awslogs_handler = AwsLogsHandler(self.context, cache_layer) - verify_as_json( - list(awslogs_handler.handle(event)), - options=Options().with_scrubber(self.scrubber), - ) - - @patch("caching.cloudwatch_log_group_cache.CloudwatchLogGroupTagsCache.__init__") - @patch("caching.cloudwatch_log_group_cache.send_forwarder_internal_metrics") - @patch.dict( - "os.environ", - { - "DD_STEP_FUNCTIONS_TRACE_ENABLED": "true", - "DD_FETCH_STEP_FUNCTIONS_TAGS": "true", - }, - ) - def test_awslogs_handler_step_functions_customized_log_group( - self, - mock_forward_metrics, - mock_cache_init, - ): - # SF customized log group - eventFromCustomizedLogGroup = { - "awslogs": { - "data": base64.b64encode( - gzip.compress( - bytes( - json.dumps( - { - "messageType": "DATA_MESSAGE", - "owner": "425362996713", - "logGroup": "test/logs", - "logStream": ( - "states/logs-to-traces-sequential/2022-11-10-15-50/7851b2d9" - ), - "subscriptionFilters": ["testFilter"], - "logEvents": [ - { - "id": ( - "37199773595581154154810589279545129148442535997644275712" - ), - "timestamp": 1668095539607, - "message": ( - '{"id": "1","type": "ExecutionStarted","details": {"input": "{}","inputDetails": {"truncated": "false"},"roleArn": "arn:aws:iam::12345678910:role/service-role/StepFunctions-test-role-a0iurr4pt"},"previous_event_id": "0","event_timestamp": "1716992192441","execution_arn": "arn:aws:states:us-east-1:12345678910:execution:StepFunction2:ccccccc-d1da-4c38-b32c-2b6b07d713fa","redrive_count": "0"}' - ), - } - ], - } - ), - "utf-8", - ) - ) - ) - } - } - mock_forward_metrics.side_effect = MagicMock() - mock_cache_init.return_value = None - cache_layer = CacheLayer("") - cache_layer._step_functions_cache.get = MagicMock( - return_value=["test_tag_key:test_tag_value"] - ) - cache_layer._cloudwatch_log_group_cache.get = MagicMock() - - awslogs_handler = AwsLogsHandler(self.context, cache_layer) - # for some reasons, the below two are needed to update the context of the handler - verify_as_json( - list(awslogs_handler.handle(eventFromCustomizedLogGroup)), - options=Options().with_scrubber(self.scrubber), - ) - def test_awslogs_handler_lambda_log(self): event = { "awslogs": { @@ -418,82 +292,6 @@ def test_get_lower_cased_lambda_function_name(self): ) -class TestParsingStepFunctionLogs(unittest.TestCase): - def setUp(self): - self.context = Context() - self.aws_handler = AwsLogsHandler(self.context, None) - - def test_get_state_machine_arn(self): - aws_attributes = AwsAttributes( - context=self.context, - log_events=[ - { - "message": json.dumps({"no_execution_arn": "xxxx/yyy"}), - } - ], - ) - - self.assertEqual(self.aws_handler.get_state_machine_arn(aws_attributes), "") - - aws_attributes = AwsAttributes( - context=self.context, - log_events=[ - { - "message": json.dumps( - { - "execution_arn": ( - "arn:aws:states:sa-east-1:425362996713:express:my-Various-States:7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - ) - } - ), - } - ], - ) - self.assertEqual( - self.aws_handler.get_state_machine_arn(aws_attributes), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - aws_attributes = AwsAttributes( - context=self.context, - log_events=[ - { - "message": json.dumps( - { - "execution_arn": ( - "arn:aws:states:sa-east-1:425362996713:express:my-Various-States/7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - ) - } - ) - } - ], - ) - - self.assertEqual( - self.aws_handler.get_state_machine_arn(aws_attributes), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - aws_attributes = AwsAttributes( - context=self.context, - log_events=[ - { - "message": json.dumps( - { - "execution_arn": ( - "arn:aws:states:sa-east-1:425362996713:express:my-Various-States\\7f653fda-c79a-430b-91e2-3f97eb87cabb:862e5d40-a457-4ca2-a3c1-78485bd94d3f" - ) - } - ) - } - ], - ) - self.assertEqual( - self.aws_handler.get_state_machine_arn(aws_attributes), - "arn:aws:states:sa-east-1:425362996713:stateMachine:my-Various-States", - ) - - class TestAwsPartitionExtraction(unittest.TestCase): def test_get_log_group_aws_partition(self): # default partition diff --git a/aws/logs_monitoring/tests/test_customized_log_group.py b/aws/logs_monitoring/tests/test_customized_log_group.py index 74fc0182a..c33e81e31 100644 --- a/aws/logs_monitoring/tests/test_customized_log_group.py +++ b/aws/logs_monitoring/tests/test_customized_log_group.py @@ -3,7 +3,6 @@ from customized_log_group import ( get_lambda_function_name_from_logstream_name, is_lambda_customized_log_group, - is_step_functions_log_group, ) @@ -60,16 +59,3 @@ def get_lambda_function_name_from_logstream_name(self): get_lambda_function_name_from_logstream_name(stepfunction_log_stream_name), None, ) - - def test_is_step_functions_log_group(self): - # Lambda logstream is false - lambda_log_stream_name = "2023/11/04/[$LATEST]4426346c2cdf4c54a74d3bd2b929fc44" - self.assertEqual(is_step_functions_log_group(lambda_log_stream_name), False) - - # SF logstream is true - step_functions_log_stream_name = ( - "states/selfmonit-statemachine/2024-11-04-15-30/00000000" - ) - self.assertEqual( - is_step_functions_log_group(step_functions_log_stream_name), True - ) diff --git a/aws/logs_monitoring/tools/integration_tests/docker-compose.yml b/aws/logs_monitoring/tools/integration_tests/docker-compose.yml index 86e6b57d1..b1953a635 100644 --- a/aws/logs_monitoring/tools/integration_tests/docker-compose.yml +++ b/aws/logs_monitoring/tools/integration_tests/docker-compose.yml @@ -30,7 +30,6 @@ services: DD_API_URL: http://recorder:8080 DD_FETCH_LAMBDA_TAGS: "${DD_FETCH_LAMBDA_TAGS:-false}" DD_FETCH_LOG_GROUP_TAGS: "${DD_FETCH_LOG_GROUP_TAGS:-false}" - DD_FETCH_STEP_FUNCTIONS_TAGS: "${DD_FETCH_STEP_FUNCTIONS_TAGS:-false}" DD_LOG_LEVEL: ${LOG_LEVEL:-info} DD_LOGS_INTAKE_URL: recorder:8080 DD_NO_SSL: "true" diff --git a/aws/logs_monitoring/tools/integration_tests/integration_tests.sh b/aws/logs_monitoring/tools/integration_tests/integration_tests.sh index 5e3745d1e..4fb8a4319 100755 --- a/aws/logs_monitoring/tools/integration_tests/integration_tests.sh +++ b/aws/logs_monitoring/tools/integration_tests/integration_tests.sh @@ -22,7 +22,6 @@ ADDITIONAL_LAMBDA=false CACHE_TEST=false DD_FETCH_LAMBDA_TAGS="false" DD_FETCH_LOG_GROUP_TAGS="false" -DD_FETCH_STEP_FUNCTIONS_TAGS="false" DD_STORE_FAILED_EVENTS="false" script_start_time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") @@ -86,7 +85,6 @@ if [ $CACHE_TEST == true ]; then SNAPSHOTS_DIR_NAME="snapshots-cache-test" DD_FETCH_LAMBDA_TAGS="true" DD_FETCH_LOG_GROUP_TAGS="true" - DD_FETCH_STEP_FUNCTIONS_TAGS="true" # Deploy test lambda function with tags AWS_LAMBDA_FUNCTION_INVOKED="cache_test_lambda" @@ -156,7 +154,6 @@ LOG_LEVEL=${LOG_LEVEL} \ SNAPSHOTS_DIR_NAME="./${SNAPSHOTS_DIR_NAME}" \ DD_FETCH_LAMBDA_TAGS=${DD_FETCH_LAMBDA_TAGS} \ DD_FETCH_LOG_GROUP_TAGS=${DD_FETCH_LOG_GROUP_TAGS} \ - DD_FETCH_STEP_FUNCTIONS_TAGS=${DD_FETCH_STEP_FUNCTIONS_TAGS} \ DD_STORE_FAILED_EVENTS=${DD_STORE_FAILED_EVENTS} \ docker compose up --build --abort-on-container-exit diff --git a/aws/logs_monitoring/tools/integration_tests/snapshots/step_functions_log.json~snapshot b/aws/logs_monitoring/tools/integration_tests/snapshots/step_functions_log.json~snapshot index ed0e1234f..40adbaa66 100644 --- a/aws/logs_monitoring/tools/integration_tests/snapshots/step_functions_log.json~snapshot +++ b/aws/logs_monitoring/tools/integration_tests/snapshots/step_functions_log.json~snapshot @@ -11,13 +11,13 @@ }, "invoked_function_arn": "arn:aws:lambda:us-east-1:012345678912:function:test_function" }, - "ddsource": "stepfunction", + "ddsource": "cloudwatch", "ddsourcecategory": "aws", "ddtags": "forwardername:test_function,forwarder_version:", "host": "/aws/vendedlogs/states/logs-to-traces-sequential-Logs", "id": "37199773595581154154810589279545129148442535997644275712", "message": "{\"id\":\"1\",\"type\":\"ExecutionStarted\",\"details\":{\"input\":\"{\"Comment\": \"Insert your JSON here\"}\",\"inputDetails\":{\"truncated\":false},\"roleArn\":\"arn:aws:iam::425362996713:role/service-role/StepFunctions-logs-to-traces-sequential-role-ccd69c03\"},\",previous_event_id\":\"0\",\"event_timestamp\":\"1668095539607\",\"execution_arn\":\"arn:aws:states:sa-east-1:425362996713:express:logs-to-traces-sequential::\"}", - "service": "stepfunction", + "service": "cloudwatch", "timestamp": 1668095539607 } ],