From 17ff79677c2fa0c34bbeb4584dcc1ee509f83565 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:15:28 +0100 Subject: [PATCH 01/15] Add nightly system-tests workflow Run the official system-tests reusable workflow daily at 4 AM GMT using dev mode against the latest commit on the default branch. Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/system-tests-nightly.yml diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml new file mode 100644 index 0000000000..4900b26976 --- /dev/null +++ b/.github/workflows/system-tests-nightly.yml @@ -0,0 +1,20 @@ +name: Nightly System Tests + +on: + workflow_dispatch: {} + schedule: + - cron: '0 4 * * *' + +jobs: + system-tests: + uses: DataDog/system-tests/.github/workflows/system-tests.yml@main + secrets: inherit + permissions: + contents: read + packages: write + with: + library: php + ref: main + scenarios_groups: tracer-release + skip_empty_scenarios: true + _system_tests_dev_mode: true From 10f7eab796fefc1c371c319da25612f6a9763f3e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:23:44 +0100 Subject: [PATCH 02/15] Enable push_to_test_optimization Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 4900b26976..7f01e409d9 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -18,3 +18,4 @@ jobs: scenarios_groups: tracer-release skip_empty_scenarios: true _system_tests_dev_mode: true + push_to_test_optimization: true From 2f73f4b49ffc6d5ede09c28e6fed089b8646b1cb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:40:31 +0100 Subject: [PATCH 03/15] Pass secrets explicitly to reusable workflow Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 7f01e409d9..5bbe185231 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -8,7 +8,9 @@ on: jobs: system-tests: uses: DataDog/system-tests/.github/workflows/system-tests.yml@main - secrets: inherit + secrets: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + TEST_OPTIMIZATION_API_KEY: ${{ secrets.DATADOG_API_KEY }} permissions: contents: read packages: write From 34d46d272af77b0f53d67a64ac0d90662d74c02f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:41:55 +0100 Subject: [PATCH 04/15] Only pass TEST_OPTIMIZATION_API_KEY secret Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 5bbe185231..527b73038d 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -9,7 +9,6 @@ jobs: system-tests: uses: DataDog/system-tests/.github/workflows/system-tests.yml@main secrets: - DD_API_KEY: ${{ secrets.DD_API_KEY }} TEST_OPTIMIZATION_API_KEY: ${{ secrets.DATADOG_API_KEY }} permissions: contents: read From 420b281664fd8d3672f9594a2a40c23dc0a58618 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:46:00 +0100 Subject: [PATCH 05/15] tmp: add pull_request trigger for testing Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 527b73038d..37682fd3d7 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -1,6 +1,7 @@ name: Nightly System Tests on: + pull_request: workflow_dispatch: {} schedule: - cron: '0 4 * * *' From 1185a365772d3b2050f94d1adf6cc49586559f0c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 15:58:33 +0100 Subject: [PATCH 06/15] Set desired_execution_time to 300s Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 37682fd3d7..16be144fce 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -20,4 +20,5 @@ jobs: scenarios_groups: tracer-release skip_empty_scenarios: true _system_tests_dev_mode: true + desired_execution_time: 300 push_to_test_optimization: true From 4af9b76884be481db76247eb70c933c87ba4b422 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 18 Feb 2026 17:42:14 +0100 Subject: [PATCH 07/15] Also pass DD_API_KEY secret Co-authored-by: Cursor --- .github/workflows/system-tests-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 16be144fce..662a9774d9 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -10,6 +10,7 @@ jobs: system-tests: uses: DataDog/system-tests/.github/workflows/system-tests.yml@main secrets: + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} TEST_OPTIMIZATION_API_KEY: ${{ secrets.DATADOG_API_KEY }} permissions: contents: read From 55526dd4400840e9f7e7a43a5b645073ee7c445e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 26 Feb 2026 16:01:30 +0100 Subject: [PATCH 08/15] ci: add system-tests Test Optimization upload and tracer-release nightly - Add after_script to .system_tests base job to upload JUnit results to Datadog Test Optimization using the existing upload script - Add tracer-release scenario group to configure_system_tests on schedule and master runs - Remove separate GitHub Actions nightly workflow Made-with: Cursor --- .github/workflows/system-tests-nightly.yml | 25 ---------------------- .gitlab/generate-package.php | 12 +++++++++++ 2 files changed, 12 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/system-tests-nightly.yml diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml deleted file mode 100644 index 662a9774d9..0000000000 --- a/.github/workflows/system-tests-nightly.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Nightly System Tests - -on: - pull_request: - workflow_dispatch: {} - schedule: - - cron: '0 4 * * *' - -jobs: - system-tests: - uses: DataDog/system-tests/.github/workflows/system-tests.yml@main - secrets: - DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} - TEST_OPTIMIZATION_API_KEY: ${{ secrets.DATADOG_API_KEY }} - permissions: - contents: read - packages: write - with: - library: php - ref: main - scenarios_groups: tracer-release - skip_empty_scenarios: true - _system_tests_dev_mode: true - desired_execution_time: 300 - push_to_test_optimization: true diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 0836af2a0d..c0c507650c 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -100,6 +100,11 @@ variables: SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi" ALLOW_MULTIPLE_CHILD_LEVELS: "false" + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_REF_NAME == "master"' + variables: + SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi,tracer-release" + - when: on_success package-oci: needs: @@ -1223,6 +1228,13 @@ when: always paths: - .cache/ + after_script: + - mkdir -p artifacts + - | + for f in system-tests/logs*/reportJunit.xml; do + [ -f "$f" ] && cp "$f" "artifacts/$(basename $(dirname "$f"))_reportJunit.xml" + done + - .gitlab/silent-upload-junit-to-datadog.sh artifacts: paths: - "system-tests/logs_parametric/" From 42bcd971b8e83bdea3444f93d3fd3dcde3b39da0 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 26 Feb 2026 16:54:49 +0100 Subject: [PATCH 09/15] ci: fix after_script by using upload script directly The silent wrapper's grep returns exit code 1 when no summary lines are found, causing the after_script to fail. Use the non-silent version with || true to prevent after_script failures. Made-with: Cursor --- .gitlab/generate-package.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index c0c507650c..2f23afc2fd 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1234,7 +1234,7 @@ for f in system-tests/logs*/reportJunit.xml; do [ -f "$f" ] && cp "$f" "artifacts/$(basename $(dirname "$f"))_reportJunit.xml" done - - .gitlab/silent-upload-junit-to-datadog.sh + - .gitlab/upload-junit-to-datadog.sh || true artifacts: paths: - "system-tests/logs_parametric/" From 50f858bf514329ecb37c289831dd8a35a2091cc3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 26 Feb 2026 17:05:30 +0100 Subject: [PATCH 10/15] ci: fix system-tests Test Optimization upload Replace the generic upload-junit-to-datadog.sh with a custom after_script that: - Uses --service system-tests (not dd-trace-php-tests) - References logs directly from system-tests/logs*/ - Uses the datadog-ci standalone binary (no npm needed) - Uses Python for JSON/zip (guaranteed available in the image) - Provides explicit error messages instead of silent exit 0 Made-with: Cursor --- .gitlab/generate-package.php | 42 +++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 2f23afc2fd..e137b39ef0 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -1229,12 +1229,44 @@ paths: - .cache/ after_script: - - mkdir -p artifacts - | - for f in system-tests/logs*/reportJunit.xml; do - [ -f "$f" ] && cp "$f" "artifacts/$(basename $(dirname "$f"))_reportJunit.xml" - done - - .gitlab/upload-junit-to-datadog.sh || true + set +e + echo "=== Uploading system-tests JUnit results to Test Optimization ===" + + # Check that there are JUnit files to upload + ls system-tests/logs*/reportJunit.xml >/dev/null 2>&1 + if [ $? -ne 0 ]; then echo "No JUnit XML files found, skipping upload"; exit 0; fi + + # Download datadog-ci standalone binary + echo "Downloading datadog-ci..." + curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" -o /tmp/datadog-ci + if [ $? -ne 0 ]; then echo "Failed to download datadog-ci"; exit 0; fi + chmod +x /tmp/datadog-ci + + # Download and extract Vault CLI + echo "Downloading Vault..." + VAULT_VERSION="1.20.0" + curl -L --fail "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip" -o /tmp/vault.zip + if [ $? -ne 0 ]; then echo "Failed to download Vault"; exit 0; fi + python3 -c "import zipfile; zipfile.ZipFile('/tmp/vault.zip').extractall('/tmp/')" + chmod +x /tmp/vault + + # Fetch API key from Vault + echo "Fetching API key from Vault..." + VAULT_JSON=$(/tmp/vault kv get --format=json "kv/k8s/gitlab-runner/dd-trace-php/datadoghq-api-key" 2>&1) + if [ $? -ne 0 ]; then echo "Failed to fetch API key from Vault: $VAULT_JSON"; exit 0; fi + export DATADOG_API_KEY=$(echo "$VAULT_JSON" | python3 -c "import sys,json; print(json.loads(sys.stdin.read())['data']['data']['key'])") + export DATADOG_SITE="datadoghq.com" + + # Upload JUnit results + echo "Uploading JUnit results..." + /tmp/datadog-ci junit upload \ + system-tests/logs*/reportJunit.xml \ + --service system-tests \ + --env ci \ + --verbose \ + --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + echo "=== Upload complete ===" artifacts: paths: - "system-tests/logs_parametric/" From 34f79e3201d1e8602bae4c39341340dfca0a1cb9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 2 Mar 2026 14:20:20 +0100 Subject: [PATCH 11/15] Run only tracer-release on schedule, it already includes other groups Made-with: Cursor --- .gitlab/generate-package.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index e137b39ef0..3be62edcc9 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -101,9 +101,9 @@ SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi" ALLOW_MULTIPLE_CHILD_LEVELS: "false" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_REF_NAME == "master"' + - if: '$CI_PIPELINE_SOURCE == "schedule"' variables: - SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi,tracer-release" + SYSTEM_TESTS_SCENARIOS_GROUPS: "tracer-release" - when: on_success package-oci: From bafec28b3cb835a8d8dbf171610346cdb0c8737d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 2 Mar 2026 14:21:52 +0100 Subject: [PATCH 12/15] Temporarily run tracer-release on all triggers for testing Made-with: Cursor --- .gitlab/generate-package.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 3be62edcc9..4538367784 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -101,10 +101,10 @@ SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi" ALLOW_MULTIPLE_CHILD_LEVELS: "false" rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' + # - if: '$CI_PIPELINE_SOURCE == "schedule"' + - when: on_success variables: SYSTEM_TESTS_SCENARIOS_GROUPS: "tracer-release" - - when: on_success package-oci: needs: From 578e0b6ed8e2d6e472db7a907bcef9208e24e268 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 5 Mar 2026 12:07:01 +0100 Subject: [PATCH 13/15] Tracer release scenario group on main --- .github/workflows/system_tests.yml | 26 ++++++++++++++++++++++++++ .gitlab/generate-package.php | 5 ----- 2 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/system_tests.yml diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml new file mode 100644 index 0000000000..49c50813fa --- /dev/null +++ b/.github/workflows/system_tests.yml @@ -0,0 +1,26 @@ +name: System Tests + +on: + push: + branches: + - main + workflow_dispatch: {} + schedule: + - cron: '00 03 * * 1-5' + +jobs: + system-tests: + uses: DataDog/system-tests/.github/workflows/system-tests.yml@main + secrets: + DD_API_KEY: ${{ secrets.DD_API_KEY }} + TEST_OPTIMIZATION_API_KEY: ${{ secrets.DD_API_KEY }} + permissions: + contents: read + packages: write + with: + library: php + ref: master + scenarios_groups: tracer-release + skip_empty_scenarios: true + _system_tests_dev_mode: true + push_to_test_optimization: true diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 4538367784..43784c3185 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -100,11 +100,6 @@ variables: SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,lib-injection,lib-injection-profiling,docker-ssi" ALLOW_MULTIPLE_CHILD_LEVELS: "false" - rules: - # - if: '$CI_PIPELINE_SOURCE == "schedule"' - - when: on_success - variables: - SYSTEM_TESTS_SCENARIOS_GROUPS: "tracer-release" package-oci: needs: From 4db06556d557b53f5a79f595055c35e20344a1d6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 5 Mar 2026 12:11:12 +0100 Subject: [PATCH 14/15] Test --- .github/workflows/system_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 49c50813fa..3035568256 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -7,6 +7,7 @@ on: workflow_dispatch: {} schedule: - cron: '00 03 * * 1-5' + pull_request: jobs: system-tests: From a65519dc59afce41c0dae00c7f609af6ea965f09 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 5 Mar 2026 12:24:37 +0100 Subject: [PATCH 15/15] Remove ref argument --- .github/workflows/system_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/system_tests.yml b/.github/workflows/system_tests.yml index 3035568256..bb6b64426e 100644 --- a/.github/workflows/system_tests.yml +++ b/.github/workflows/system_tests.yml @@ -20,7 +20,6 @@ jobs: packages: write with: library: php - ref: master scenarios_groups: tracer-release skip_empty_scenarios: true _system_tests_dev_mode: true