diff --git a/.gitlab/input_files/build.yaml.tpl b/.gitlab/input_files/build.yaml.tpl index 737d746e..527308bb 100644 --- a/.gitlab/input_files/build.yaml.tpl +++ b/.gitlab/input_files/build.yaml.tpl @@ -1,3 +1,5 @@ +{{- $e2e_region := "us-west-2" -}} + variables: CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-js CI_DOCKER_TARGET_VERSION: latest @@ -7,6 +9,7 @@ stages: - test - sign - publish + - e2e default: retry: @@ -94,6 +97,7 @@ integration test ({{ $runtime.name }}): - RUNTIME_PARAM={{ $runtime.node_major_version }} ./scripts/run_integration_tests.sh {{ range $environment := (ds "environments").environments }} +{{ $dotenv := print $runtime.name "_" $environment.name ".env" }} {{ if or (eq $environment.name "prod") }} sign layer ({{ $runtime.name }}): @@ -126,10 +130,15 @@ publish layer {{ $environment.name }} ({{ $runtime.name }}): tags: ["arch:amd64"] image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} rules: + - if: '"{{ $environment.name }}" == "sandbox" && $REGION == "{{ $e2e_region }}"' + when: on_success - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' when: manual allow_failure: true - if: '$CI_COMMIT_TAG =~ /^v.*/' + artifacts: + reports: + dotenv: {{ $dotenv }} needs: {{ if or (eq $environment.name "prod") }} - sign layer ({{ $runtime.name }}) @@ -154,7 +163,7 @@ publish layer {{ $environment.name }} ({{ $runtime.name }}): before_script: - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh script: - - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} .gitlab/scripts/publish_layers.sh + - STAGE={{ $environment.name }} NODE_VERSION={{ $runtime.node_version }} DOTENV={{ $dotenv }} .gitlab/scripts/publish_layers.sh {{- end }} @@ -203,3 +212,27 @@ publish npm package: - mkdir -p datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} - cp .layers/datadog_lambda_node*.zip datadog_lambda_js-{{ if eq $environment.name "prod"}}signed-{{ end }}bundle-${CI_JOB_ID} {{ end }} + +e2e-test: + stage: e2e + trigger: + project: DataDog/serverless-e2e-tests + strategy: depend + variables: + LANGUAGES_SUBSET: node + {{- range (ds "runtimes").runtimes }} + {{- $version := print (.name | strings.Trim "node") }} + NODEJS_{{ $version }}_VERSION: $NODE_{{ $version }}_VERSION + {{- end }} + needs: {{ range (ds "runtimes").runtimes }} + - "publish layer sandbox ({{ .name }}): [{{ $e2e_region }}]" + {{- end }} + + +e2e-test-status: + stage: e2e + image: registry.ddbuild.io/images/docker:20.10-py3 + tags: ["arch:amd64"] + timeout: 3h + script: + - .gitlab/scripts/poll_e2e.sh diff --git a/.gitlab/scripts/poll_e2e.sh b/.gitlab/scripts/poll_e2e.sh new file mode 100755 index 00000000..4d7652e6 --- /dev/null +++ b/.gitlab/scripts/poll_e2e.sh @@ -0,0 +1,38 @@ +curl -OL "binaries.ddbuild.io/dd-source/authanywhere/LATEST/authanywhere-linux-amd64" && mv "authanywhere-linux-amd64" /bin/authanywhere && chmod +x /bin/authanywhere + +BTI_CI_API_TOKEN=$(authanywhere --audience rapid-devex-ci) + +BTI_RESPONSE=$(curl --silent --request GET \ + --header "$BTI_CI_API_TOKEN" \ + --header "Content-Type: application/vnd.api+json" \ + "https://bti-ci-api.us1.ddbuild.io/internal/ci/gitlab/token?owner=DataDog&repository=datadog-lambda-js") + +GITLAB_TOKEN=$(echo "$BTI_RESPONSE" | jq -r '.token // empty') + +URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" + +echo "Fetching E2E job status from: $URL" + +while true; do + RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$URL") + E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .downstream_pipeline.status') + echo -n "E2E job status: $E2E_JOB_STATUS, " + if [ "$E2E_JOB_STATUS" == "success" ]; then + echo "✅ E2E tests completed successfully" + exit 0 + elif [ "$E2E_JOB_STATUS" == "failed" ]; then + echo "❌ E2E tests failed" + exit 1 + elif [ "$E2E_JOB_STATUS" == "running" ]; then + echo "⏳ E2E tests are still running, retrying in 2 minutes..." + elif [ "$E2E_JOB_STATUS" == "canceled" ]; then + echo "🚫 E2E tests were canceled" + exit 1 + elif [ "$E2E_JOB_STATUS" == "skipped" ]; then + echo "⏭️ E2E tests were skipped" + exit 0 + else + echo "❓ Unknown E2E test status: $E2E_JOB_STATUS, retrying in 2 minutes..." + fi + sleep 120 +done \ No newline at end of file diff --git a/.gitlab/scripts/publish_layers.sh b/.gitlab/scripts/publish_layers.sh index efca7f11..3bd888a4 100755 --- a/.gitlab/scripts/publish_layers.sh +++ b/.gitlab/scripts/publish_layers.sh @@ -146,6 +146,8 @@ fi while [ $latest_version -lt $VERSION ]; do latest_version=$(publish_layer $REGION $layer $aws_cli_node_version_key $layer_path) printf "[$REGION] Published version $latest_version for layer $layer in region $REGION\n" + latest_arn=$(aws lambda get-layer-version --layer-name $layer --version-number $latest_version --region $REGION --query 'LayerVersionArn' --output text) + printf "[$REGION] Published arn $latest_arn\n" # This shouldn't happen unless someone manually deleted the latest version, say 28, and # then tries to republish 28 again. The published version would actually be 29, because @@ -156,4 +158,12 @@ while [ $latest_version -lt $VERSION ]; do fi done +if [ -n "$DOTENV" ]; then + printf "[$REGION] Exporting layer version to $DOTENV file...\n" + node_version=$NODE_VERSION + major_version="${node_version:0:2}" + echo "NODE_${major_version}_VERSION=$latest_arn" >> "$DOTENV" + cat "$DOTENV" +fi + printf "[$REGION] Finished publishing layers...\n\n"