Skip to content

feat(go-forwarder): add minimal lambda enrichment #1178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ndakkoune merged 5 commits into from
Jul 22, 2026
Merged

feat(go-forwarder): add minimal lambda enrichment #1178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d4c5c90
feat(go-forwarder): add minimal lambda enrichment
ndakkoune Jul 20, 2026
293ec3e
fix
ndakkoune Jul 21, 2026
60cd6a1
clean separation
ndakkoune Jul 21, 2026
973d61e
add service and tag from lmbda name
ndakkoune Jul 22, 2026
8aeb533
one service tag
ndakkoune Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions aws/logs_monitoring_go/cmd/forwarder/testdata/cloudwatch_lambda_coldstart.golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,57 +10,66 @@
},
"body": [
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35486831490800643125153606102923171443962457178576257024",
"timestamp": 1591284559098,
"message": "START RequestId: db275f87-a934-471a-8980-b63bf4dc1beb Version: $LATEST\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/04/[$LATEST]af2b1e1843b84a2d80c67840ae3ffa72",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
},
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35486831490867545360749197972347778598780402263094198273",
"timestamp": 1591284559101,
"message": "END RequestId: db275f87-a934-471a-8980-b63bf4dc1beb\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/04/[$LATEST]af2b1e1843b84a2d80c67840ae3ffa72",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
},
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35486831490867545360749197972347778598780402263094198274",
"timestamp": 1591284559101,
"message": "REPORT RequestId: db275f87-a934-471a-8980-b63bf4dc1beb\\tDuration: 1.76 ms\\tBilled Duration: 100 ms\\tMemory Size: 128 MB\\tMax Memory Used: 48 MB\\tInit Duration: 120.96 ms\\t\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/04/[$LATEST]af2b1e1843b84a2d80c67840ae3ffa72",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
}
]
Expand Down
27 changes: 18 additions & 9 deletions aws/logs_monitoring_go/cmd/forwarder/testdata/cloudwatch_lambda_custom_log_group.golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,57 +10,66 @@
},
"body": [
{
"host": "/aws/vendedlogs/states/anyLogGroupName",
"host": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup",
"id": "35311576111948622874033876462979853992919938886093242368",
"timestamp": 1583425836114,
"message": "2020-03-05T16:30:36.113Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Patched console output with trace context\"}\n",
"service": "lambda",
"service": "test-customized-loggroup",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:test-customized-loggroup,env:none,service:test-customized-loggroup",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/vendedlogs/states/anyLogGroupName",
"logStream": "2020/03/05/test-customized-loggroup[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup"
}
},
{
"host": "/aws/vendedlogs/states/anyLogGroupName",
"host": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup",
"id": "35311576111948622874033876462979853992919938886093242369",
"timestamp": 1583425836114,
"message": "2020-03-05T16:30:36.114Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"autoPatchHTTP\":true,\"tracerInitialized\":true,\"status\":\"debug\",\"message\":\"datadog:Not patching HTTP libraries\"}\n",
"service": "lambda",
"service": "test-customized-loggroup",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:test-customized-loggroup,env:none,service:test-customized-loggroup",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/vendedlogs/states/anyLogGroupName",
"logStream": "2020/03/05/test-customized-loggroup[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup"
}
},
{
"host": "/aws/vendedlogs/states/anyLogGroupName",
"host": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup",
"id": "35311576111948622874033876462979853992919938886093242370",
"timestamp": 1583425836114,
"message": "2020-03-05T16:30:36.114Z\tf08bb4c8-d6b2-4f05-ac17-af7e2ba005fb\tDEBUG\t[dd.trace_id=3172564172058669914 dd.span_id=14292093692483532556] {\"status\":\"debug\",\"message\":\"datadog:Reading trace context from env var Root=1-5e61292c-cc1229a4dfbeae1043928548;Parent=c657b77d9514f70c;Sampled=1\"}\n",
"service": "lambda",
"service": "test-customized-loggroup",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:test-customized-loggroup,env:none,service:test-customized-loggroup",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/vendedlogs/states/anyLogGroupName",
"logStream": "2020/03/05/test-customized-loggroup[$LATEST]20bddfd5a2dc4c6b97ac02800eae90d0",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:test-customized-loggroup"
}
}
]
Expand Down
36 changes: 24 additions & 12 deletions aws/logs_monitoring_go/cmd/forwarder/testdata/cloudwatch_lambda_timeout.golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,75 +10,87 @@
},
"body": [
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35496429375792603298393743017356257146982675867810398208",
"timestamp": 1591714943146,
"message": "START RequestId: 7c9567b5-107b-4a6c-8798-0157ac21db52 Version: $LATEST\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
},
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35496429442806342619978265557671090556291002193281548289",
"timestamp": 1591714946151,
"message": "END RequestId: 7c9567b5-107b-4a6c-8798-0157ac21db52\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
},
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35496429442806342619978265557671090556291002193281548290",
"timestamp": 1591714946151,
"message": "REPORT RequestId: 7c9567b5-107b-4a6c-8798-0157ac21db52\\tDuration: 3003.16 ms\\tBilled Duration: 3000 ms\\tMemory Size: 128 MB\\tMax Memory Used: 48 MB\\tInit Duration: 127.02 ms\\t\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
},
{
"host": "/aws/lambda/storms-cloudwatch-event",
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35496429442806342619978265557671090556291002193281548291",
"timestamp": 1591714946151,
"message": "2020-06-09T15:02:26.150Z 7c9567b5-107b-4a6c-8798-0157ac21db52 Task timed out after 3.00 seconds\\n\\n",
"service": "lambda",
"service": "storms-cloudwatch-event",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "",
"ddtags": "functionname:storms-cloudwatch-event,env:none,service:storms-cloudwatch-event",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
}
]
Expand Down
34 changes: 34 additions & 0 deletions aws/logs_monitoring_go/cmd/forwarder/testdata/cloudwatch_lambda_with_dd_message.golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"headers": {
"Content-Encoding": "gzip",
"Content-Type": "application/json",
"Dd-Api-Key": "abcdefghijklmnopqrstuvwxyz012345",
"Dd-Evp-Origin": "aws_forwarder",
"Dd-Evp-Origin-Version": "6.0",
"Dd-Storage-Tag": "cloudwatch",
"User-Agent": "Go-http-client/1.1"
},
"body": [
{
"host": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event",
"id": "35496429442806342619978265557671090556291002193281548291",
"timestamp": 1591714946151,
"message": "{\"message\":\"hello world\"}",
"service": "custom_service",
"ddsource": "lambda",
"ddsourcecategory": "aws",
"ddtags": "custom_tag1:value1,custom_tag2:value2,functionname:storms-cloudwatch-event,env:none,service:custom_service",
"aws": {
"invoked_function_arn": "arn:aws:lambda:us-east-1:123456789012:function:forwarder",
"awslogs": {
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"owner": "601427279990"
}
},
"lambda": {
"arn": "arn:aws:lambda:us-east-1:123456789012:function:storms-cloudwatch-event"
}
}
]
}
15 changes: 15 additions & 0 deletions aws/logs_monitoring_go/cmd/forwarder/testdata/cloudwatch_lambda_with_dd_message.input.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"messageType": "DATA_MESSAGE",
"owner": "601427279990",
"logGroup": "/aws/lambda/storms-cloudwatch-event",
"logStream": "2020/06/09/[$LATEST]b249865adaaf4fad80f95f8ad09725b8",
"subscriptionFilters": [
"myevent"
],
"logEvents": [{
"id": "35496429442806342619978265557671090556291002193281548291",
"timestamp": 1591714946151,
"message": "{\"message\": \"hello world\", \"ddtags\": \"service:custom_service,custom_tag1:value1,custom_tag2:value2\"}\n"
}
]
}
26 changes: 21 additions & 5 deletions aws/logs_monitoring_go/internal/handling/cloudwatch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ const (
logStreamCloudtrail = "_CloudTrail_"
)

const envTag = "env"

// Custom log groups use the log stream format: YYYY/MM/DD/<function_name>[<function_version>][<execution_environment_GUID>]
var lambdaLogStreamRegex = regexp.MustCompile(
`^\d{4}/[01]\d/[0-3]\d/[\w.-]{1,75}\[(\$LATEST|[\w-]{1,129})\][0-9a-f]{32}$`,
`^\d{4}/[01]\d/[0-3]\d/([\w.-]{1,75})\[(?:\$LATEST|[\w-]{1,129})\][0-9a-f]{32}$`,
)

type cloudwatchHandler struct {
Expand Down Expand Up @@ -130,18 +132,28 @@ func (h cloudwatchHandler) newCloudwatchBaseEntry(data events.CloudwatchLogsData
Owner: data.Owner,
},
}
source := cloudwatchSource(strings.ToLower(logGroup), logStream)

entry := model.NewLogEntry()
entry.Source = cmp.Or(h.cfg.Source, CloudwatchSource(strings.ToLower(logGroup), logStream))
entry.Source = cmp.Or(h.cfg.Source, source)
entry.Host = cmp.Or(h.cfg.Host, logGroup)
entry.Metadata = metadata

if entry.Source == sourceLambda {
Comment thread
ge0Aja marked this conversation as resolved.
enrichLambdaLog(&entry, lambdaOrigin.ARN, logGroup, logStream)
if !h.cfg.Tags.Has(envTag) {
entry.Tags.Add(envTag, "none")
}
}

return entry
}

func (h cloudwatchHandler) newCloudwatchLogEntry(event events.CloudwatchLogsLogEvent, entry model.LogEntry) model.LogEntry {
tags, service, message := extractFromMessage(event.Message)
entry.Service = cmp.Or(h.cfg.Service, service, entry.Source)
entry.Tags = slices.Concat(tags, h.cfg.Tags)

entry.Service = cmp.Or(service, entry.Service, h.cfg.Service, entry.Source)
entry.Tags = slices.Concat(tags, entry.Tags, h.cfg.Tags)
entry.Message = message
entry.ID = event.ID
entry.Timestamp = event.Timestamp
Expand All @@ -150,10 +162,14 @@ func (h cloudwatchHandler) newCloudwatchLogEntry(event events.CloudwatchLogsLogE
entry.Host = cloudtrailHost(event.Message)
}

if entry.Lambda != nil && !entry.Tags.Has("service") {
entry.Tags.Add("service", entry.Service)
}

return entry
}

func CloudwatchSource(logGroup, logStream string) string {
func cloudwatchSource(logGroup, logStream string) string {
if strings.HasPrefix(logStream, logStreamStepFunction) {
return sourceStepFunction
}
Expand Down
Loading
Loading