Add OpenTelemetry/DataDog API logging [prod]

.github/dependabot.yml

@@ -12,20 +12,50 @@ updates:
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/cla-landing-page" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+    open-pull-requests-limit: 3
+    ignore:
+      - dependency-name: "serverless"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
+      - dependency-name: "serverless-domain-manager"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/cla-backend" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+    open-pull-requests-limit: 3
+    ignore:
+      - dependency-name: "serverless"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
+      - dependency-name: "serverless-domain-manager"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
   - package-ecosystem: "pip" # See documentation for possible values
     directory: "/cla-backend" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+    open-pull-requests-limit: 3
+    ignore:
+      - dependency-name: "serverless"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
+      - dependency-name: "serverless-domain-manager"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/cla-backend-go" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+    open-pull-requests-limit: 3
+    ignore:
+      - dependency-name: "serverless"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
+      - dependency-name: "serverless-domain-manager"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
   - package-ecosystem: "gomod" # See documentation for possible values
     directory: "/cla-backend-go" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+    open-pull-requests-limit: 3
+    ignore:
+      - dependency-name: "serverless"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]
+      - dependency-name: "serverless-domain-manager"
+        update-types: ["version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch"]

.github/workflows/build-pr.yml

@@ -16,6 +16,7 @@ permissions:
 env:
   AWS_REGION: us-east-1
   STAGE: dev
+  DD_VERSION: ${{ github.sha }}
 
 jobs:
   build-test-lint:

.github/workflows/codeql-analysis.yml

@@ -5,54 +5,42 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [main, ]
+    branches: [main]
   pull_request:
-    # The branches below must be a subset of the branches above
     branches: [main]
   schedule:
     - cron: '0 5 * * 4'
 
 jobs:
   analyse:
-    name: Analyse
+    name: Analyze (${{ matrix.language }})
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
 
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    # Note: git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.
-    #- run: git checkout HEAD^2
-    #  if: ${{ github.event_name == 'pull_request' }}
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      # Override language selection by uncommenting this and choosing your languages
-      # with:
-      #   languages: go, javascript, csharp, python, cpp, java
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['go', 'python', 'javascript']
 
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v4
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/deploy-dev.yml

@@ -16,6 +16,7 @@ permissions:
 env:
   AWS_REGION: us-east-1
   STAGE: dev
+  DD_VERSION: ${{ github.sha }}
 
 jobs:
   build-deploy-dev:

.github/workflows/deploy-prod.yml

@@ -18,6 +18,7 @@ permissions:
 env:
   AWS_REGION: us-east-1
   STAGE: prod
+  DD_VERSION: ${{ github.sha }}
 
 jobs:
   build-deploy-prod:

.github/workflows/yarn-scan-backend-go-pr.yml

@@ -9,6 +9,10 @@ on:
   pull_request:
     branches:
       - dev
+    paths:
+      - "cla-backend-go/package.json"
+      - "cla-backend-go/yarn.lock"
+      - ".github/workflows/yarn-scan-backend-go-pr.yml"
 
 jobs:
   yarn-scan-backend-go-pr:
@@ -25,4 +29,5 @@ jobs:
       - name: Yarn Audit
         working-directory: cla-backend-go
         run: |
-          yarn audit
+          yarn audit --json > audit.json || true
+          node ../scripts/yarn-audit-filter.mjs audit.json ../.yarn-audit-allowlist.json

.github/workflows/yarn-scan-backend-pr.yml

@@ -9,6 +9,10 @@ on:
   pull_request:
     branches:
       - dev
+    paths:
+      - "cla-backend/package.json"
+      - "cla-backend/yarn.lock"
+      - ".github/workflows/yarn-scan-backend-pr.yml"
 
 jobs:
   yarn-scan-backend-pr:
@@ -25,4 +29,5 @@ jobs:
       - name: Yarn Audit
         working-directory: cla-backend
         run: |
-          yarn audit
+          yarn audit --json > audit.json || true
+          node ../scripts/yarn-audit-filter.mjs audit.json ../.yarn-audit-allowlist.json

.gitignore

@@ -266,3 +266,7 @@ cla-backend/python-api.log
 cla-backend/python-api.err
 cla-backend-go/golang-api.err
 cla-backend-go/golang-api.log
+utils/otel_dd_go/otel_dd
+audit.json
+spans*.json
+api_usage.csv

.yarn-audit-allowlist.json

@@ -0,0 +1,9 @@
+{
+  "minSeverity": "high",
+  "allowlist": [
+    1111997
+  ],
+  "notes": {
+    "1111997": "aws-sdk v2 advisory flagged as 'No patch available' in our current baseline; accepted until migration."
+  }
+}

cla-backend-go/bootstrap

@@ -0,0 +1,21 @@
+#!/bin/sh
+set -eu
+
+# AWS Lambda custom runtime entrypoint (provided.al2 / provided.al2023).
+# AWS executes /var/task/bootstrap. The configured Lambda "Handler" is exposed
+# via the $_HANDLER env var. We exec that handler binary (which is our Go
+# executable) so we don't need to rename every binary to 'bootstrap'.
+
+if [ -z "${_HANDLER:-}" ]; then
+  echo "bootstrap: _HANDLER is not set" >&2
+  exit 1
+fi
+
+HANDLER_PATH="/var/task/${_HANDLER}"
+if [ ! -x "${HANDLER_PATH}" ]; then
+  echo "bootstrap: handler '${HANDLER_PATH}' not found or not executable" >&2
+  ls -la /var/task >&2 || true
+  exit 1
+fi
+
+exec "${HANDLER_PATH}"

cla-backend-go/cmd/server.go

@@ -81,6 +81,7 @@ import (
 
 	"github.com/linuxfoundation/easycla/cla-backend-go/api_logs"
 	"github.com/linuxfoundation/easycla/cla-backend-go/signatures"
+	"github.com/linuxfoundation/easycla/cla-backend-go/telemetry"
 	v2Signatures "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures"
 
 	ini "github.com/linuxfoundation/easycla/cla-backend-go/init"
@@ -146,8 +147,52 @@ type combinedRepo struct {
 	projects_cla_groups.Repository
 }
 
+const (
+	envDDBAPILogging         = "DDB_API_LOGGING"
+	envOtelDatadogAPILogging = "OTEL_DATADOG_API_LOGGING"
+)
+
+// parseBoolish parses common "boolean-ish" env var values.
+// Returns (value, ok). ok=false means "unknown/invalid".
+func parseBoolish(v string) (bool, bool) {
+	s := strings.TrimSpace(strings.ToLower(v))
+	switch s {
+	case "1", "true", "yes", "y", "on":
+		return true, true
+	case "0", "false", "no", "n", "off":
+		return false, true
+	default:
+		return false, false
+	}
+}
+
+// enabledByEnvOrStage implements:
+// - if env var set to true/1/yes -> enabled
+// - if env var set to false/0/no -> disabled
+// - if env var unset/empty -> defaultByStage[idx]
+// - idx 0 = dev/default stage, index 1 = prod stage
+func enabledByEnvOrStage(envVar, stage string, defaultByStage [2]bool) bool {
+	if raw, ok := os.LookupEnv(envVar); ok && strings.TrimSpace(raw) != "" {
+		if b, ok2 := parseBoolish(raw); ok2 {
+			return b
+		}
+		log.Warnf("LG:api-log-flag-invalid:%s value=%q (falling back to STAGE default)", envVar, raw)
+	}
+	st := strings.TrimSpace(strings.ToLower(stage))
+	if st == "prod" || st == "production" {
+		return defaultByStage[1]
+	}
+	// dev and all non-prod stages default to enabled
+	return defaultByStage[0]
+}
+
 // apiPathLoggerWithDB creates a middleware that logs API requests to DynamoDB
 func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) http.Handler {
+	// No-op when API logging is disabled. This prevents nil deref panics if the middleware
+	// remains in the handler chain but repo creation is skipped.
+	if apiLogsRepo == nil {
+		return func(next http.Handler) http.Handler { return next }
+	}
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			log.Infof("LG:api-request-path:%s", r.URL.Path)
@@ -177,6 +222,8 @@ func apiPathLoggerWithDB(apiLogsRepo api_logs.Repository) func(http.Handler) htt
 }
 
 // server function called by environment specific server functions
+//
+//nolint:gocyclo
 func server(localMode bool) http.Handler {
 	f := logrus.Fields{
 		"functionName": "cmd.server",
@@ -208,6 +255,28 @@ func server(localMode bool) http.Handler {
 
 	stage := viper.GetString("STAGE")
 	dynamodbRegion := ini.GetProperty("DYNAMODB_AWS_REGION")
+	ddbAPILoggingEnabled := enabledByEnvOrStage(envDDBAPILogging, stage, [2]bool{true, false})
+	otelDatadogEnabled := enabledByEnvOrStage(envOtelDatadogAPILogging, stage, [2]bool{true, true})
+
+	// Initialize OTel SDK -> Datadog Lambda Extension (OTLP) once at cold start.
+	// If init fails, disable OTel logging but never fail startup.
+	if otelDatadogEnabled {
+		version := strings.TrimSpace(Commit)
+		if strings.TrimSpace(version) == "" {
+			version = Version
+		}
+		if strings.TrimSpace(version) == "" {
+			version = "unknown"
+		}
+		if er := telemetry.InitDatadogOTel(telemetry.DatadogOTelConfig{
+			Stage:   stage,
+			Service: "easycla-backend",
+			Version: version,
+		}); er != nil {
+			log.Infof("LG:otel-datadog-disabled err=%v", er)
+			otelDatadogEnabled = false
+		}
+	}
 
 	log.WithFields(f).Infof("Service %s starting...", ini.ServiceName)
 
@@ -221,6 +290,8 @@ func server(localMode bool) http.Handler {
 		log.Infof("Golang OS               : %s", runtime.GOOS)
 		log.Infof("Golang Arch             : %s", runtime.GOARCH)
 		log.Infof("DYANAMODB_AWS_REGION    : %s", dynamodbRegion)
+		log.Infof("DDB_API_LOGGING         : %t", ddbAPILoggingEnabled)
+		log.Infof("OTEL_DATADOG_API_LOGGING: %t", otelDatadogEnabled)
 		log.Infof("GH_ORG_VALIDATION       : %t", githubOrgValidation)
 		log.Infof("COMPANY_USER_VALIDATION : %t", companyUserValidation)
 		log.Infof("STAGE                   : %s", stage)
@@ -239,6 +310,8 @@ func server(localMode bool) http.Handler {
 		f["companyUserValidation"] = companyUserValidation
 		f["stage"] = stage
 		f["serviceHost"] = host
+		f["ddbAPILogging"] = ddbAPILoggingEnabled
+		f["otelDatadog"] = otelDatadogEnabled
 		log.WithFields(f).Info("config")
 	}
 
@@ -305,7 +378,14 @@ func server(localMode bool) http.Handler {
 	approvalListRepo := approval_list.NewRepository(awsSession, stage)
 	v1CompanyRepo := v1Company.NewRepository(awsSession, stage)
 	eventsRepo := events.NewRepository(awsSession, stage)
-	apiLogsRepo := api_logs.NewRepository(stage, dynamodb.New(awsSession))
+
+	var apiLogsRepo api_logs.Repository
+	if ddbAPILoggingEnabled {
+		apiLogsRepo = api_logs.NewRepository(stage, dynamodb.New(awsSession))
+	} else {
+		apiLogsRepo = nil
+	}
+
 	v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage)
 	v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo)
 	metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, v1ProjectClaGroupRepo)
@@ -497,6 +577,12 @@ func server(localMode bool) http.Handler {
 				v2API.Serve(middlewareSetupfunc), v2SwaggerSpec.BasePath()),
 			configFile.AllowedOrigins)
 	}
+
+	// OTel/Datadog (OTLP -> Datadog Lambda Extension) - enabled by flag
+	if otelDatadogEnabled {
+		apiHandler = telemetry.WrapHTTPHandler(apiHandler)
+	}
+
 	return apiHandler
 }