From 0b185eb526be5345dab82be99db9bf2574812454 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Tue, 10 Jun 2025 17:23:11 +0200 Subject: [PATCH 1/7] chore(benchmarks): add PR performance quality gate --- .gitlab-ci.yml | 1 + .gitlab/benchmarks.yml | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5c7ae83..25d8ed4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,5 @@ stages: - benchmarks + - benchmarks-gate include: ".gitlab/benchmarks.yml" diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 983995a1..5b705f58 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -1,11 +1,11 @@ variables: - BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-cpp + BENCHMARKS_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:dd-trace-cpp benchmarks: stage: benchmarks when: on_success tags: ["runner:apm-k8s-tweaked-metal"] - image: $BASE_CI_IMAGE + image: $BENCHMARKS_CI_IMAGE interruptible: true timeout: 15m script: @@ -19,6 +19,7 @@ benchmarks: - "./steps/post-pr-comment.sh || :" artifacts: name: "reports" + when: always paths: - reports/ expire_in: 3 months @@ -30,3 +31,28 @@ benchmarks: KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" + +benchmarks-pr-comment: + stage: benchmarks-gate + needs: [ benchmarks ] + when: on_success + tags: ["arch:amd64"] + image: $BENCHMARKS_CI_IMAGE + script: + - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) + - bp-runner bp-runner.pr-comment.yml --debug + allow_failure: true + variables: + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp + +check-big-regressions: + stage: benchmarks-gate + needs: [ benchmarks ] + when: on_success + tags: ["arch:amd64"] + image: $BENCHMARKS_CI_IMAGE + script: + - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) + - bp-runner bp-runner.fail-on-regression.yml --debug + variables: + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp \ No newline at end of file From c83f7483995f68b2cfe5a5a3b5226ed31df59291 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Tue, 10 Jun 2025 17:33:44 +0200 Subject: [PATCH 2/7] fix: right folder --- .gitlab/benchmarks.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 5b705f58..6cf9f4b3 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -32,19 +32,6 @@ benchmarks: KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" -benchmarks-pr-comment: - stage: benchmarks-gate - needs: [ benchmarks ] - when: on_success - tags: ["arch:amd64"] - image: $BENCHMARKS_CI_IMAGE - script: - - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) - - bp-runner bp-runner.pr-comment.yml --debug - allow_failure: true - variables: - KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp - check-big-regressions: stage: benchmarks-gate needs: [ benchmarks ] @@ -52,7 +39,7 @@ check-big-regressions: tags: ["arch:amd64"] image: $BENCHMARKS_CI_IMAGE script: - - cd platform && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) + - cd reports && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) - bp-runner bp-runner.fail-on-regression.yml --debug variables: KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp \ No newline at end of file From 68c8d64b829f4ed1043f759b2f8e9fe652f70980 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Wed, 11 Jun 2025 14:52:26 +0200 Subject: [PATCH 3/7] set artifacts dir --- .gitlab/benchmarks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 6cf9f4b3..26b7a3c8 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -39,6 +39,7 @@ check-big-regressions: tags: ["arch:amd64"] image: $BENCHMARKS_CI_IMAGE script: + - export ARTIFACTS_DIR="$(pwd)/reports" - cd reports && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) - bp-runner bp-runner.fail-on-regression.yml --debug variables: From c558ae3e788d1fa4265d8d91acef1341e3d70573 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Wed, 11 Jun 2025 14:59:08 +0200 Subject: [PATCH 4/7] fix attempt 1 --- .gitlab/benchmarks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index 26b7a3c8..db658737 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -39,7 +39,8 @@ check-big-regressions: tags: ["arch:amd64"] image: $BENCHMARKS_CI_IMAGE script: - - export ARTIFACTS_DIR="$(pwd)/reports" + - export ARTIFACTS_DIR="$(pwd)/reports/" + - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - cd reports && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) - bp-runner bp-runner.fail-on-regression.yml --debug variables: From 07796a0a81a3d2ed83c78ae04146d7637cba99d0 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Wed, 11 Jun 2025 15:08:58 +0200 Subject: [PATCH 5/7] fix attempt 2 --- .gitlab-ci.yml | 2 +- .gitlab/benchmarks.yml | 31 +++++++++++++++++++------------ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 25d8ed4c..7f9babbc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ stages: - benchmarks - - benchmarks-gate + - benchmarks-report include: ".gitlab/benchmarks.yml" diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml index db658737..92d70780 100644 --- a/.gitlab/benchmarks.yml +++ b/.gitlab/benchmarks.yml @@ -11,12 +11,13 @@ benchmarks: script: - export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :) - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - - git clone --branch dd-trace-cpp https://github.com/DataDog/benchmarking-platform /platform && cd /platform - - ./steps/capture-hardware-software-info.sh - - ./steps/run-benchmarks.sh - - ./steps/analyze-results.sh - - "./steps/upload-results-to-s3.sh || :" - - "./steps/post-pr-comment.sh || :" + - git clone --branch dd-trace-cpp https://github.com/DataDog/benchmarking-platform /platform + - export PATH="$PATH:/platform/steps" + - capture-hardware-software-info.sh + - run-benchmarks.sh + - analyze-results.sh + - "upload-results-to-s3.sh || :" + - "post-pr-comment.sh || :" artifacts: name: "reports" when: always @@ -33,15 +34,21 @@ benchmarks: FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" check-big-regressions: - stage: benchmarks-gate + stage: benchmarks-report needs: [ benchmarks ] when: on_success + allow_failure: false tags: ["arch:amd64"] image: $BENCHMARKS_CI_IMAGE - script: - - export ARTIFACTS_DIR="$(pwd)/reports/" - - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - - cd reports && (git init && git remote add origin https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform && git pull origin dd-trace-cpp) - - bp-runner bp-runner.fail-on-regression.yml --debug + script: | + export ARTIFACTS_DIR="$(pwd)/reports/" + if [[ -n "$CI_JOB_TOKEN" ]]; + then + git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" + fi + git clone --branch dd-trace-cpp https://github.com/DataDog/benchmarking-platform /platform + export PATH="$PATH:/platform/steps" + + bp-runner /platform/bp-runner.fail-on-regression.yml --debug variables: KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: dd-trace-cpp \ No newline at end of file From 7eb83c572d3ef3a23dd2fc4eb1f72a5ee907aec6 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Thu, 12 Jun 2025 10:19:07 +0200 Subject: [PATCH 6/7] test: add performance regression --- src/datadog/tracer.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 8849c082..4ed16738 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -170,6 +170,15 @@ Span Tracer::create_span(const SpanConfig& config) { auto defaults = config_manager_->span_defaults(); auto span_data = std::make_unique(); span_data->apply_config(*defaults, config, clock_); + + // PERFORMANCE REGRESSION: Add expensive computation to simulate heavy workload + // This will cause significant performance degradation in span creation + volatile std::uint64_t regression_work = 0; + for (int i = 0; i < 1000000; ++i) { + regression_work += (i * i) % 997; // Use a prime modulus for more work + regression_work += regression_work * 31; // Additional computation + } + span_data->trace_id = generator_->trace_id(span_data->start); span_data->span_id = span_data->trace_id.low; span_data->parent_id = 0; From 0c98cc67acd7346ebca929919433dac2fd44ee79 Mon Sep 17 00:00:00 2001 From: Louis Tricot Date: Thu, 12 Jun 2025 10:26:14 +0200 Subject: [PATCH 7/7] the regression was not big enough --- src/datadog/tracer.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 4ed16738..941a91e0 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -174,7 +174,11 @@ Span Tracer::create_span(const SpanConfig& config) { // PERFORMANCE REGRESSION: Add expensive computation to simulate heavy workload // This will cause significant performance degradation in span creation volatile std::uint64_t regression_work = 0; - for (int i = 0; i < 1000000; ++i) { + for (int i = 0; i < 1000000000; ++i) { + regression_work += (i * i) % 997; // Use a prime modulus for more work + regression_work += regression_work * 31; // Additional computation + } + for (int i = 0; i < 1000000000; ++i) { regression_work += (i * i) % 997; // Use a prime modulus for more work regression_work += regression_work * 31; // Additional computation }