[AWSX] feat(logs forwarder): Remove StepFunctions logic

aws/logs_monitoring/README.md

@@ -784,9 +784,6 @@ To test different patterns against your logs, turn on [debug logs](#troubleshoot
 `DD_FETCH_LOG_GROUP_TAGS`
 : [DEPRECATED, use DD_ENRICH_CLOUDWATCH_TAGS] Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics, and traces. If set to true, permission `logs:ListTagsForResource` will be automatically added to the Lambda execution IAM role.
 
-`DD_FETCH_STEP_FUNCTIONS_TAGS`
-: Let the Forwarder fetch Step Functions tags using GetResources API calls and apply them to logs and traces (if Step Functions tracing is enabled). If set to true, permission `tag:GetResources` will be automatically added to the Lambda execution IAM role.
-
 `DD_STEP_FUNCTION_TRACE_ENABLED`
 : Set to true to enable tracing for all Step Functions.
 

aws/logs_monitoring/caching/cache_layer.py

@@ -1,5 +1,4 @@
 from caching.cloudwatch_log_group_cache import CloudwatchLogGroupTagsCache
-from caching.step_functions_cache import StepFunctionsTagsCache
 from caching.s3_tags_cache import S3TagsCache
 from caching.lambda_cache import LambdaTagsCache
 
@@ -8,7 +7,6 @@ class CacheLayer:
     def __init__(self, prefix):
         self._cloudwatch_log_group_cache = CloudwatchLogGroupTagsCache(prefix)
         self._s3_tags_cache = S3TagsCache(prefix)
-        self._step_functions_cache = StepFunctionsTagsCache(prefix)
         self._lambda_cache = LambdaTagsCache(prefix)
 
     def get_cloudwatch_log_group_tags_cache(self):
@@ -17,8 +15,5 @@ def get_cloudwatch_log_group_tags_cache(self):
     def get_s3_tags_cache(self):
         return self._s3_tags_cache
 
-    def get_step_functions_tags_cache(self):
-        return self._step_functions_cache
-
     def get_lambda_tags_cache(self):
         return self._lambda_cache

aws/logs_monitoring/customized_log_group.py

@@ -26,11 +26,6 @@ def is_lambda_customized_log_group(logstream_name):
     )
 
 
-# For both default and customzied Step Functions log groups, the log_stream starts with "states/"
-def is_step_functions_log_group(logstream_name):
-    return logstream_name.startswith("states/")
-
-
 def get_lambda_function_name_from_logstream_name(logstream_name):
     try:
         # Not match the pattern for customized Lambda log group

aws/logs_monitoring/logs/datadog_http_client.py

@@ -56,6 +56,9 @@ class DatadogHTTPClient(object):
     if storage_tag != "":
         _HEADERS["DD-STORAGE-TAG"] = storage_tag
 
+    if os.environ.get("DD_STEP_FUNCTIONS_TRACE_ENABLED", "false").lower() == "true":
+        _HEADERS["DD-STEP-FUNCTIONS-TRACE-ENABLED"] = "true"
+
     def __init__(
         self, host, port, no_ssl, skip_ssl_validation, api_key, scrubber, timeout=10
     ):

aws/logs_monitoring/settings.py

@@ -294,10 +294,6 @@ def is_api_key_valid():
     "DD_FETCH_LAMBDA_TAGS", default="false", boolean=True
 )
 
-DD_FETCH_STEP_FUNCTIONS_TAGS = get_env_var(
-    "DD_FETCH_STEP_FUNCTIONS_TAGS", default="false", boolean=True
-)
-
 
 DD_ENRICH_S3_TAGS = get_env_var("DD_ENRICH_S3_TAGS", default="true", boolean=True)
 
@@ -328,10 +324,6 @@ def get_fetch_lambda_tags():
     return DD_FETCH_LAMBDA_TAGS
 
 
-def get_fetch_step_functions_tags():
-    return DD_FETCH_STEP_FUNCTIONS_TAGS
-
-
 def get_enrich_s3_tags():
     return DD_ENRICH_S3_TAGS
 
@@ -367,9 +359,6 @@ def get_enrich_cloudwatch_tags():
 DD_S3_LAMBDA_CACHE_FILENAME = "lambda.json"
 DD_S3_LAMBDA_CACHE_LOCK_FILENAME = "lambda.lock"
 
-DD_S3_STEP_FUNCTIONS_CACHE_FILENAME = "step-functions-cache.json"
-DD_S3_STEP_FUNCTIONS_CACHE_LOCK_FILENAME = "step-functions-cache.lock"
-
 DD_S3_TAGS_CACHE_FILENAME = "s3.json"
 DD_S3_TAGS_CACHE_LOCK_FILENAME = "s3.lock"
 
@@ -378,7 +367,6 @@ def get_enrich_cloudwatch_tags():
 DD_TAGS_CACHE_TTL_SECONDS = int(get_env_var("DD_TAGS_CACHE_TTL_SECONDS", default=300))
 DD_S3_CACHE_LOCK_TTL_SECONDS = 60
 GET_RESOURCES_LAMBDA_FILTER = "lambda"
-GET_RESOURCES_STEP_FUNCTIONS_FILTER = "states"
 GET_RESOURCES_S3_FILTER = "s3:bucket"
 
 

aws/logs_monitoring/steps/enums.py

@@ -9,7 +9,6 @@ class AwsEventSource(Enum):
     LAMBDA = "lambda"
     S3 = "s3"
     SNS = "sns"
-    STEPFUNCTION = "stepfunction"
     WAF = "waf"
 
     def __str__(self):

aws/logs_monitoring/steps/handlers/awslogs_handler.py

@@ -3,13 +3,11 @@
 import json
 import logging
 import os
-import re
 from io import BufferedReader, BytesIO
 
 from customized_log_group import (
     get_lambda_function_name_from_logstream_name,
     is_lambda_customized_log_group,
-    is_step_functions_log_group,
 )
 from settings import DD_CUSTOM_TAGS, DD_HOST, DD_SOURCE
 from steps.common import (
@@ -98,9 +96,6 @@ def set_source(self, event, metadata, aws_attributes):
         # Need to place the handling of customized log group at the bottom so that it can correct the source for some edge cases
         if is_lambda_customized_log_group(log_stream):
             metadata[DD_SOURCE] = str(AwsEventSource.LAMBDA)
-        # Regardless of whether the log group is customized, the corresponding log stream starts with 'states/'."
-        if is_step_functions_log_group(log_stream):
-            metadata[DD_SOURCE] = str(AwsEventSource.STEPFUNCTION)
 
     def add_cloudwatch_tags_from_cache(self, metadata, aws_attributes):
         log_group_arn = aws_attributes.get_log_group_arn()
@@ -128,51 +123,6 @@ def set_host(self, metadata, aws_attributes):
         match metadata_source:
             case AwsEventSource.CLOUDWATCH:
                 metadata[DD_HOST] = log_group
-            case AwsEventSource.STEPFUNCTION:
-                self.handle_step_function_source(metadata, aws_attributes)
-
-    def handle_verified_access_source(self, metadata, aws_attributes):
-        try:
-            message = json.loads(aws_attributes.get_log_events()[0].get("message"))
-            metadata[DD_HOST] = message.get("http_request").get("url").get("hostname")
-        except Exception as e:
-            logger.debug("Unable to set verified-access log host: %s" % e)
-
-    def handle_step_function_source(self, metadata, aws_attributes):
-        state_machine_arn = self.get_state_machine_arn(aws_attributes)
-        if not state_machine_arn:
-            return
-
-        metadata[DD_HOST] = state_machine_arn
-        formatted_stepfunctions_tags = (
-            self.cache_layer.get_step_functions_tags_cache().get(state_machine_arn)
-        )
-        if len(formatted_stepfunctions_tags) > 0:
-            metadata[DD_CUSTOM_TAGS] = (
-                ",".join(formatted_stepfunctions_tags)
-                if not metadata[DD_CUSTOM_TAGS]
-                else metadata[DD_CUSTOM_TAGS]
-                + ","
-                + ",".join(formatted_stepfunctions_tags)
-            )
-
-        if os.environ.get("DD_STEP_FUNCTIONS_TRACE_ENABLED", "false").lower() == "true":
-            metadata[DD_CUSTOM_TAGS] = ",".join(
-                [metadata.get(DD_CUSTOM_TAGS, [])]
-                + ["dd_step_functions_trace_enabled:true"]
-            )
-
-    def get_state_machine_arn(self, aws_attributes):
-        try:
-            message = json.loads(aws_attributes.get_log_events()[0].get("message"))
-            if message.get("execution_arn") is not None:
-                execution_arn = message["execution_arn"]
-                arn_tokens = re.split(r"[:/\\]", execution_arn)
-                arn_tokens[5] = "stateMachine"
-                return ":".join(arn_tokens[:7])
-        except Exception as e:
-            logger.debug("Unable to get state_machine_arn: %s" % e)
-        return ""
 
     # Lambda logs can be from either default or customized log group
     def process_lambda_logs(self, metadata, aws_attributes):

aws/logs_monitoring/template.yaml

@@ -107,13 +107,6 @@ Parameters:
       - true
       - false
     Description: (DEPRECATED in favor of DdEnrichCloudwatchTags) Let the forwarder fetch Log Group tags using ListTagsLogGroup and apply them to logs, metrics and traces. If set to true, permission logs:ListTagsLogGroup will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.list_tags_log_group_api_call metric for each API call made.
-  DdFetchStepFunctionsTags:
-    Type: String
-    Default: true
-    AllowedValues:
-      - true
-      - false
-    Description: Let the forwarder fetch Step Functions tags using GetResources API calls and apply them to logs, metrics and traces. If set to true, permission tag:GetResources will be automatically added to the Lambda execution IAM role. The tags are cached in memory and S3 so that they'll only be fetched when the function cold starts or when the TTL (1 hour) expires. The forwarder increments the aws.lambda.enhanced.get_resources_api_calls metric for each API call made.
   DdFetchS3Tags:
     Type: String
     Default: false
@@ -366,7 +359,6 @@ Conditions:
   SetDdSkipSslValidation: !Equals [!Ref DdSkipSslValidation, true]
   SetDdFetchLambdaTags: !Equals [!Ref DdFetchLambdaTags, true]
   SetDdFetchLogGroupTags: !Equals [!Ref DdFetchLogGroupTags, true]
-  SetDdFetchStepFunctionsTags: !Equals [!Ref DdFetchStepFunctionsTags, true]
   CreateS3Bucket: !And
     - !Or
       - !Equals [!Ref DdFetchLogGroupTags, true]
@@ -402,7 +394,6 @@ Conditions:
     - !Equals [!Ref DdForwarderBucketsAccessLogsTarget, ""]
   ShouldDdFetchTags: !Or
     - !Equals [!Ref DdFetchLambdaTags, true]
-    - !Equals [!Ref DdFetchStepFunctionsTags, true]
     - !Equals [!Ref DdFetchS3Tags, true]
   SetForwarderBucket: !Or
     - !Condition CreateS3Bucket
@@ -523,10 +514,6 @@ Resources:
             - SetDdFetchLogGroupTags
             - !Ref DdFetchLogGroupTags
             - !Ref AWS::NoValue
-          DD_FETCH_STEP_FUNCTIONS_TAGS: !If
-            - SetDdFetchStepFunctionsTags
-            - !Ref DdFetchStepFunctionsTags
-            - !Ref AWS::NoValue
           DD_NO_SSL: !If
             - SetDdNoSsl
             - !Ref DdNoSsl
@@ -1166,7 +1153,6 @@ Metadata:
           - DdEnrichCloudwatchTags
           - DdFetchLambdaTags
           - DdFetchLogGroupTags
-          - DdFetchStepFunctionsTags
           - DdFetchS3Tags
           - DdStepFunctionsTraceEnabled
           - DdEnhancedMetrics