diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abc0ce93586..0969ea78462 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ appsec-trigger: strategy: depend variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID - GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/msgpack-c + GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c profiler-trigger: stage: tests @@ -92,6 +92,6 @@ package-trigger: strategy: depend variables: PARENT_PIPELINE_ID: $CI_PIPELINE_ID - GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/msgpack-c + GIT_SUBMODULE_PATHS: libdatadog appsec/third_party/cpp-base64 appsec/third_party/libddwaf appsec/third_party/libddwaf-rust appsec/third_party/msgpack-c NIGHTLY_BUILD: $NIGHTLY_BUILD RELIABILITY_ENV_BRANCH: $RELIABILITY_ENV_BRANCH diff --git a/.gitlab/build-appsec-helper-rust.sh b/.gitlab/build-appsec-helper-rust.sh new file mode 100755 index 00000000000..fbe223edbac --- /dev/null +++ b/.gitlab/build-appsec-helper-rust.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -e -o pipefail + +MAKE_JOBS=${MAKE_JOBS:-$(nproc)} + +mkdir -p appsec_$(uname -m) + +git config --global --add safe.directory '*' + +cd appsec/helper-rust + +export CARGO_TARGET_DIR=/tmp/cargo-target +RUST_TARGET=$(uname -m)-unknown-linux-musl + +# Build using nightly toolchain with unstable features +# -Z build-std: Rebuild std library for musl +# -Z build-std-features=llvm-libunwind: Use LLVM libunwind instead of libgcc_s +cargo +nightly-"$RUST_TARGET" build \ + --release \ + -Zhost-config \ + -Ztarget-applies-to-host \ + --target "$RUST_TARGET" + +# Remove musl libc dependency using patchelf (makes binary work on both musl and glibc) +BINARY_PATH="/tmp/cargo-target/$RUST_TARGET/release/libddappsec_helper_rust.so" +ARCH=$(uname -m) +if [ "$ARCH" = "x86_64" ]; then + patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true +elif [ "$ARCH" = "aarch64" ]; then + patchelf --remove-needed libc.musl-aarch64.so.1 "$BINARY_PATH" 2>/dev/null || true +fi + +# Copy to output +cp -v "$BINARY_PATH" "../../appsec_$(uname -m)/libddappsec-helper-rust.so" + +# Run tests +cargo +nightly-"$RUST_TARGET" test \ + --release \ + -Zhost-config \ + -Ztarget-applies-to-host \ + --target "$RUST_TARGET" diff --git a/.gitlab/generate-appsec.php b/.gitlab/generate-appsec.php index c90ccc9ea1f..8e8844bbf72 100644 --- a/.gitlab/generate-appsec.php +++ b/.gitlab/generate-appsec.php @@ -92,7 +92,7 @@ -DDD_APPSEC_TESTING=ON -DBOOST_CACHE_PREFIX=$CI_PROJECT_DIR/boost-cache" - make -j 4 xtest -"appsec integration tests": +.appsec_integration_tests: stage: test image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal # TODO: use a proper docker image with java pre-installed? tags: [ "docker-in-docker:amd64" ] @@ -100,7 +100,42 @@ KUBERNETES_CPU_REQUEST: 8 KUBERNETES_MEMORY_REQUEST: 24Gi KUBERNETES_MEMORY_LIMIT: 30Gi + DOCKER_LOOPBACK_SIZE: 30G ARCH: amd64 + HELPER_RUST_FLAG: "" + before_script: + + + script: + - apt update && apt install -y openjdk-17-jre + - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true + - | + cd appsec/tests/integration + CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz + if [ -f "$CACHE_PATH" ]; then + echo "Loading cache from $CACHE_PATH" + TERM=dumb ./gradlew loadCaches --info + fi + + TERM=dumb ./gradlew $targets --info -Pbuildscan --scan $HELPER_RUST_FLAG + TERM=dumb ./gradlew saveCaches --info + after_script: + - mkdir -p "${CI_PROJECT_DIR}/artifacts" + - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \; + - .gitlab/upload-junit-to-datadog.sh "test.source.file:appsec" + artifacts: + reports: + junit: "artifacts/**/test-results/**/TEST-*.xml" + paths: + - "artifacts/" + when: "always" + cache: + - key: "appsec int test cache" + paths: + - appsec/tests/integration/build/*.tar.gz + +"appsec integration tests": + extends: .appsec_integration_tests parallel: matrix: - targets: @@ -126,12 +161,40 @@ - test8.4-release-zts - test8.5-release - test8.5-release-zts + - test8.5-release-musl + +"appsec integration tests (helper-rust)": + extends: .appsec_integration_tests + variables: + HELPER_RUST_FLAG: "-PuseHelperRust" + parallel: + matrix: + - targets: + - test7.4-release + - test8.1-release + - test8.3-debug + - test8.4-release-zts + - test8.5-release-musl + +"helper-rust build and test": + stage: test + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal + tags: [ "docker-in-docker:amd64" ] + interruptible: true + rules: + - if: $CI_COMMIT_BRANCH == "master" + interruptible: false + - when: on_success + variables: + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_REQUEST: 8Gi + KUBERNETES_MEMORY_LIMIT: 10Gi + ARCH: amd64 before_script: script: - apt update && apt install -y openjdk-17-jre - - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true - | cd appsec/tests/integration CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz @@ -139,12 +202,142 @@ echo "Loading cache from $CACHE_PATH" TERM=dumb ./gradlew loadCaches --info fi + # Build and test helper-rust (includes formatting check and cargo test) + TERM=dumb ./gradlew testHelperRust --info -Pbuildscan --scan + TERM=dumb ./gradlew saveCaches --info + cache: + - key: "appsec int test cache" + paths: + - appsec/tests/integration/build/*.tar.gz - TERM=dumb ./gradlew $targets --info -Pbuildscan --scan +"helper-rust code coverage": + stage: test + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal + tags: [ "docker-in-docker:amd64" ] + interruptible: true + rules: + - if: $CI_COMMIT_BRANCH == "master" + interruptible: false + - when: on_success + variables: + KUBERNETES_CPU_REQUEST: 4 + KUBERNETES_MEMORY_REQUEST: 8Gi + KUBERNETES_MEMORY_LIMIT: 10Gi + ARCH: amd64 + before_script: + + + script: + - apt update && apt install -y openjdk-17-jre + - | + echo "Installing codecov CLI" + curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import + CODECOV_VERSION=0.6.1 + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig + gpgv codecov.SHA256SUM.sig codecov.SHA256SUM + shasum -a 256 -c codecov.SHA256SUM + rm codecov.SHA256SUM.sig codecov.SHA256SUM + chmod +x codecov + mv codecov /usr/local/bin/codecov + - | + echo "Installing vault for codecov token" + curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip + unzip vault.zip + mv vault /usr/local/bin/vault + rm vault.zip + - | + cd appsec/tests/integration + CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz + if [ -f "$CACHE_PATH" ]; then + echo "Loading cache from $CACHE_PATH" + TERM=dumb ./gradlew loadCaches --info + fi + # Run unit tests with coverage instrumentation + TERM=dumb ./gradlew coverageHelperRust --info -Pbuildscan --scan + TERM=dumb ./gradlew saveCaches --info + - | + echo "Extracting coverage data from Docker volume" + mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust + docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-unit.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-unit.lcov + - | + echo "Uploading helper-rust unit test coverage to codecov" + cd "$CI_PROJECT_DIR" + CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token) + codecov -t "$CODECOV_TOKEN" -n helper-rust-unit -F helper-rust-unit -v -f appsec/helper-rust/coverage-unit.lcov + artifacts: + paths: + - appsec/helper-rust/coverage-unit.lcov + when: always + cache: + - key: "appsec int test cache" + paths: + - appsec/tests/integration/build/*.tar.gz + +"helper-rust integration coverage": + stage: test + image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:24.0.4-gbi-focal + tags: [ "docker-in-docker:amd64" ] + interruptible: true + rules: + - if: $CI_COMMIT_BRANCH == "master" + interruptible: false + - when: on_success + variables: + KUBERNETES_CPU_REQUEST: 8 + KUBERNETES_MEMORY_REQUEST: 24Gi + KUBERNETES_MEMORY_LIMIT: 30Gi + ARCH: amd64 + before_script: + + + script: + - apt update && apt install -y openjdk-17-jre + - | + echo "Installing codecov CLI" + curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --keyring trustedkeys.gpg --import + CODECOV_VERSION=0.6.1 + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM + curl -Os https://uploader.codecov.io/v${CODECOV_VERSION}/linux/codecov.SHA256SUM.sig + gpgv codecov.SHA256SUM.sig codecov.SHA256SUM + shasum -a 256 -c codecov.SHA256SUM + rm codecov.SHA256SUM.sig codecov.SHA256SUM + chmod +x codecov + mv codecov /usr/local/bin/codecov + - | + echo "Installing vault for codecov token" + curl -o vault.zip https://releases.hashicorp.com/vault/1.20.0/vault_1.20.0_linux_amd64.zip + unzip vault.zip + mv vault /usr/local/bin/vault + rm vault.zip + - | + cd appsec/tests/integration + CACHE_PATH=build/php-appsec-volume-caches-${ARCH}.tar.gz + if [ -f "$CACHE_PATH" ]; then + echo "Loading cache from $CACHE_PATH" + TERM=dumb ./gradlew loadCaches --info + fi + # Build helper-rust with coverage instrumentation + TERM=dumb ./gradlew buildHelperRustWithCoverage --info -Pbuildscan --scan + # Run integration tests with coverage-instrumented binary + TERM=dumb ./gradlew test8.3-debug --info -Pbuildscan --scan -PuseHelperRustCoverage + # Generate coverage report from profraw files + TERM=dumb ./gradlew generateHelperRustIntegrationCoverage --info -Pbuildscan --scan TERM=dumb ./gradlew saveCaches --info + - | + echo "Extracting coverage data from Docker volume" + mkdir -p "$CI_PROJECT_DIR"/appsec/helper-rust + docker run --rm -v php-helper-rust-coverage:/vol alpine cat /vol/coverage-integration.lcov > "$CI_PROJECT_DIR"/appsec/helper-rust/coverage-integration.lcov + - | + echo "Uploading helper-rust integration test coverage to codecov" + cd "$CI_PROJECT_DIR" + CODECOV_TOKEN=$(vault kv get --format=json kv/k8s/gitlab-runner/dd-trace-php/codecov | jq -r .data.data.token) + codecov -t "$CODECOV_TOKEN" -n helper-rust-integration -F helper-rust-integration -v -f appsec/helper-rust/coverage-integration.lcov after_script: - mkdir -p "${CI_PROJECT_DIR}/artifacts" - - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \; + - find appsec/tests/integration/build/test-results -name "*.xml" -exec cp --parents '{}' "${CI_PROJECT_DIR}/artifacts/" \; || true - cp -r appsec/tests/integration/build/test-logs "${CI_PROJECT_DIR}/artifacts/" 2>/dev/null || true - .gitlab/silent-upload-junit-to-datadog.sh "test.source.file:appsec" artifacts: @@ -152,7 +345,8 @@ junit: "artifacts/**/test-results/**/TEST-*.xml" paths: - "artifacts/" - when: "always" + - appsec/helper-rust/coverage-integration.lcov + when: always cache: - key: "appsec int test cache" paths: diff --git a/.gitlab/generate-package.php b/.gitlab/generate-package.php index 0836af2a0d9..8dbacfebee3 100644 --- a/.gitlab/generate-package.php +++ b/.gitlab/generate-package.php @@ -280,6 +280,24 @@ paths: - "appsec_*" +"compile appsec helper rust": + stage: appsec + image: "registry.ddbuild.io/images/mirror/datadog/dd-appsec-php-ci:nginx-fpm-php-8.5-release-musl" + tags: [ "arch:$ARCH" ] + needs: [ "prepare code" ] + parallel: + matrix: + - ARCH: ["amd64", "arm64" ] + variables: + MAKE_JOBS: 12 + KUBERNETES_CPU_REQUEST: 12 + KUBERNETES_MEMORY_REQUEST: 8Gi + KUBERNETES_MEMORY_LIMIT: 12Gi + script: .gitlab/build-appsec-helper-rust.sh + artifacts: + paths: + - "appsec_*" + "pecl build": stage: tracing image: "registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-7.4_bookworm-6" @@ -613,13 +631,20 @@ } ?> - # Compile appsec helper + # Compile appsec helper (C++) - job: "compile appsec helper" parallel: matrix: - ARCH: "" artifacts: true + # Compile appsec helper (Rust) + - job: "compile appsec helper rust" + parallel: + matrix: + - ARCH: "" + artifacts: true + @@ -690,6 +715,11 @@ matrix: - ARCH: "" artifacts: true + - job: "compile appsec helper rust" + parallel: + matrix: + - ARCH: "" + artifacts: true - job: "compile loader: [linux-gnu, ]" artifacts: true - job: "compile loader: [linux-musl, ]" diff --git a/.gitmodules b/.gitmodules index bcef3459a1c..16212a50047 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,7 @@ [submodule "tea/benchmarks/google-benchmark"] path = tea/benchmarks/google-benchmark url = https://github.com/google/benchmark.git +[submodule "appsec/third_party/libddwaf-rust"] + path = appsec/third_party/libddwaf-rust + url = https://github.com/DataDog/libddwaf-rust.git + branch = glopes/v2 diff --git a/appsec/.dockerignore b/appsec/.dockerignore new file mode 100644 index 00000000000..e31462ec01a --- /dev/null +++ b/appsec/.dockerignore @@ -0,0 +1,17 @@ +# Ignore everything by default +* + +# Include only what's needed for helper-rust portable build +!helper-rust/ +!third_party/libddwaf-rust/ + +# Exclude build artifacts and caches from helper-rust +helper-rust/target/ +helper-rust/.cargo/ +helper-rust/build/ +helper-rust/.idea/ +helper-rust/*.log + +# Exclude build artifacts from libddwaf-rust +third_party/libddwaf-rust/target/ +third_party/libddwaf-rust/.cargo/ diff --git a/appsec/cmake/helper.cmake b/appsec/cmake/helper.cmake index 2c8043bd7f0..b550d5bd6d5 100644 --- a/appsec/cmake/helper.cmake +++ b/appsec/cmake/helper.cmake @@ -29,7 +29,7 @@ target_compile_options(helper_objects PRIVATE -ftls-model=global-dynamic) target_link_libraries(helper_objects PUBLIC libddwaf_objects pthread spdlog cpp-base64 msgpack_c rapidjson_appsec boost_system zlibstatic) -target_compile_options(helper_objects PRIVATE -Wno-gnu-anonymous-struct -Wno-nested-anon-types) +target_compile_options(helper_objects PRIVATE -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-error=pedantic -Wno-error=deprecated-declarations) add_library(ddappsec-helper SHARED src/helper/main.cpp diff --git a/appsec/helper-rust/.cargo/config.toml b/appsec/helper-rust/.cargo/config.toml new file mode 100644 index 00000000000..fec60eba16b --- /dev/null +++ b/appsec/helper-rust/.cargo/config.toml @@ -0,0 +1,22 @@ +target-applies-to-host=false + +[profile.release] +debug = 2 +lto = "thin" +codegen-units = 1 + +[unstable] +build-std = ["std", "panic_unwind"] +build-std-features = ["llvm-libunwind"] + +[target.aarch64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static", "-C", "link-arg=-Wl,--compress-debug-sections=zlib"] + +[target.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static", "-C", "link-arg=-Wl,--compress-debug-sections=zlib"] + +[host.aarch64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] + +[host.x86_64-unknown-linux-musl] +rustflags = ["-C", "target-feature=-crt-static"] diff --git a/appsec/helper-rust/.gitignore b/appsec/helper-rust/.gitignore new file mode 100644 index 00000000000..ea8c4bf7f35 --- /dev/null +++ b/appsec/helper-rust/.gitignore @@ -0,0 +1 @@ +/target diff --git a/appsec/helper-rust/CLAUDE.md b/appsec/helper-rust/CLAUDE.md new file mode 100644 index 00000000000..f56a1be73e6 --- /dev/null +++ b/appsec/helper-rust/CLAUDE.md @@ -0,0 +1,361 @@ +# helper-rust - AppSec Helper Rust Rewrite + +## Purpose + +This project is a Rust rewrite of the Datadog AppSec helper for PHP, which provides application security monitoring and runtime protection capabilities. The helper is a library loaded into sidecar that: + +- Executes the Datadog WAF (Web Application Firewall) on request data +- Handles remote configuration updates for security rules +- Collects and submits telemetry metrics +- Provides RASP (Runtime Application Self-Protection) capabilities +- Extracts API schema information +- Manages security actions (blocking, redirecting, recording events) + +## Related Code + +- **C++ helper (original)**: `../src/helper/` + - Reference implementation for all features + - Telemetry definitions: `../src/helper/telemetry.hpp`, `tags.hpp`, `metrics.hpp` + - Remote config: `../src/helper/remote_config/` + - WAF integration: `../src/helper/subscriber/waf.cpp` + - Protocol: `../src/helper/network/proto.hpp` + +- **PHP extension**: `../src/extension` (integration point) + +- **libddwaf Rust bindings**: `../third_party/libddwaf-rust/` (path dependency) + +- **libddwaf C++ library**: `../third_party/libddwaf/` (built separately for LIBDDWAF\_PREFIX) + +## Architecture + +### Current Architecture (Standalone Binary) + +``` +PHP Extension + ↓ (Unix socket + msgpack) +helper-rust (loaded by sidecar) + ↓ (FFI) +libddwaf +``` + +### Target Architecture (Sidecar Integration) + +Eventually, we'll move to: + +``` +PHP Extension + ↓ (sidecar protocol) +Sidecar ← helper-rust (embedded) + ↓ (FFI) +libddwaf +``` + +## Key Components + +Core modules: +- **src/lib.rs** - C FFI entry point (`appsec_helper_main()`), initialization, runtime management +- **src/server.rs** - Unix socket server that accepts client connections +- **src/client.rs** - Client connection handler, request processing, WAF execution orchestration +- **src/service.rs** - Service management, maintains WAF instances per service configuration +- **src/config.rs** - Configuration management (from environment variables) +- **src/rc.rs** - Remote configuration reader using shared memory polling +- **src/rc_notify.rs** - Remote configuration callback system to receive updates from sidecar +- **src/telemetry.rs** - Telemetry traits and definitions for metrics and logs +- **src/ffi.rs** - FFI helpers and symbol resolution for calling sidecar functions +- **src/lock.rs** - Lock file and abstract socket uniqueness enforcement + +Client sub-modules: +- **src/client/protocol.rs** - Msgpack protocol codec for PHP extension communication +- **src/client/log.rs** - Logging utilities +- **src/client/metrics.rs** - Request-level metrics collection +- **src/client/attributes.rs** - Request attributes processing + +Service sub-modules: +- **src/service/updateable_waf.rs** - Thread-safe WAF wrapper with atomic updates +- **src/service/config_manager.rs** - ASM feature configuration from remote config +- **src/service/limiter.rs** - Rate limiting for trace submission +- **src/service/sampler.rs** - Trace sampling +- **src/service/metrics.rs** - Service-level metrics +- **src/service/waf_diag.rs** - WAF diagnostics collection +- **src/service/waf_ruleset.rs** - WAF ruleset management and loading + +Telemetry sub-modules: +- **src/telemetry/sidecar.rs** - Sidecar FFI telemetry submission +- **src/telemetry/error_tel_ctx.rs** - Error telemetry context management +- **src/telemetry/tel_aware_logger.rs** - Logger that integrates with telemetry system + +## Building + +### Using Gradle (Integration Tests) + +The helper-rust is built via Gradle for integration testing. From `tests/integration/`: + +```bash +# Build helper-rust and libddwaf +./gradlew buildHelperRust --info + +# The output files are in the php-helper-rust Docker volume: +# - libddappsec-helper-rust.so +# - libddwaf.so +``` + +The build task: +1. Builds libddwaf as a shared library using CMake +2. Builds helper-rust with Cargo, setting `LIBDDWAF_PREFIX` to point to the libddwaf installation + +## Development Notes + +- Uses Tokio for async runtime +- Built with Rust 2021 edition +- Unit tests should be added for new features using `#[test]` + +### Integration Tests + +Integration tests run via Gradle from `tests/integration/`: + +```bash +./gradlew :test8.3-debug -PuseHelperRust --info \ + --tests "com.datadog.appsec.php.integration.Apache2FpmTests.test sampling priority" +``` + +(if omitting -PuseHelperRust, the C++ helper implementation will be used) + +Logs for the helper are available at `tests/integration/build/test-logs/{helper,appsec}.log` + +**Important**: When validating changes, run tests on **both glibc and musl** systems: +- Glibc (Debian): `test8.3-debug` or other standard test targets +- Musl (Alpine): `test8.5-release-musl` + +The helper-rust binary is built to work universally on both platforms. Example: +```bash +# Test on glibc +./gradlew test8.3-debug --tests "*NginxFpmTests*" -PuseHelperRust + +# Test on musl +./gradlew test8.5-release-musl --tests "*NginxFpmTests*" -PuseHelperRust +``` + +### Test Targets by PHP Version/Variant + +Some test classes require specific PHP versions or ZTS (Zend Thread Safety) variants: + + | Test Class | Required Target | Condition | + | ------------ | ----------------- | ----------- | + | FrankenphpClassicTests | test8.4-release-zts | PHP 8.4 + ZTS | + | FrankenphpWorkerTests | test8.4-release-zts | PHP 8.4 + ZTS | + | Laravel8xTests | test7.4-debug | PHP 7.4, non-ZTS | + | Symfony62Tests | test8.1-debug | PHP 8.1, non-ZTS | + | RoadRunnerTests | test7.4-debug (or later) | PHP >= 7.4, non-ZTS | + +Available gradle targets follow the pattern: `test{version}-{variant}` where: +- version: 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2, 8.3, 8.4, 8.5 +- variant: debug, release, release-zts, release-musl (for Alpine/musl testing: 8.5-release only) + +## Style + +- Do not add comments describing what you're doing. Instead, if not obvious, explain the rationale behind some code. +- Put the public API at the top and move implementation types and implementation details to the bottom. As a subsidiary rule, prefer to put functions immediately after their caller, if possible. + +## System Tests + +System tests are located in `../../../system-tests/` (relative to dd-trace-php root) and provide end-to-end testing for the tracer and AppSec functionality. + +### Setting Up Binaries for System Tests + +Before running system tests with the Rust helper, copy the required binaries: + +```bash +# Build helper-rust via Gradle +cd tests/integration +./gradlew buildHelperRust --info + +# Extract binaries from Docker volume +docker run -i --rm -v php-helper-rust:/vol alpine cat /vol/libddappsec-helper-rust.so > ../../system-tests/binaries/libddappsec-helper.so +docker run -i --rm -v php-helper-rust:/vol alpine cat /vol/libddwaf.so > ../../system-tests/binaries/libddwaf.so + +# If there were modifications in ddtrace or the extension relative to the latest origin/master: +./gradlew buildAppsec-8.0-release buildTracer-8.0-release --info + +docker run -i --rm -v php-appsec-8.0-release:/appsec alpine cat /appsec/ddappsec.so > ../system-tests/binaries/ddappsec.so +docker run -i --rm -v php-tracer-8.0-release:/tracer alpine cat /tracer/build_extension/.libs/ddtrace.so > ../system-tests/binaries/ddtrace.so +``` + +### Running System Tests + +From the `system-tests/` directory: + +```bash +# Build the PHP weblog image +./build.sh php + +# Run a specific scenario +./run.sh + +# Run a scenario group +./run.sh _SCENARIOS +# e.g., ./run.sh APPSEC_SCENARIOS +``` + +### Running Specific Tests + +```bash +# Run a specific test file +./run.sh SCENARIO_NAME tests/path_to_test.py + +# Run a specific test class +./run.sh tests/appsec/waf/test_addresses.py::Test_BodyJson + +# Run a specific test method +./run.sh tests/appsec/waf/test_addresses.py::Test_BodyJson::test_body_json + +# Run tests matching a pattern +./run.sh SCENARIO_NAME -k "test_pattern" +``` + +### PHP/AppSec Relevant Scenarios + +Core AppSec scenarios: +- `APPSEC_BLOCKING` - Misc tests for AppSec blocking +- `APPSEC_CUSTOM_RULES` - Test custom AppSec rules file +- `APPSEC_MISSING_RULES` - Test missing AppSec rules file +- `APPSEC_CORRUPTED_RULES` - Test corrupted AppSec rules file +- `APPSEC_CUSTOM_OBFUSCATION` - Test custom obfuscation parameters +- `APPSEC_RATE_LIMITER` - Tests with low rate trace limit +- `APPSEC_WAF_TELEMETRY` - WAF telemetry tests + +API Security scenarios: +- `APPSEC_API_SECURITY` - API Security with schema types +- `APPSEC_API_SECURITY_RC` - API Security Remote config +- `APPSEC_API_SECURITY_NO_RESPONSE_BODY` - API Security without response body +- `APPSEC_API_SECURITY_WITH_SAMPLING` - API Security with sampling + +Standalone/APM opt-out scenarios: +- `APPSEC_STANDALONE` - AppSec standalone mode +- `APPSEC_STANDALONE_API_SECURITY` - API Security in standalone mode + +RASP scenarios: +- `APPSEC_RASP` - RASP with internal server +- `APPSEC_RASP_NON_BLOCKING` - RASP with non-blocking rules +- `APPSEC_STANDALONE_RASP` - RASP standalone (tracing disabled) + +Remote configuration scenarios: +- `APPSEC_BLOCKING_FULL_DENYLIST` - Rules from remote config +- `APPSEC_RUNTIME_ACTIVATION` - AppSec activation via remote config +- `REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES` - RC with ASM features +- `REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD` - RC with ASM DD backend + +User event scenarios: +- `APPSEC_AUTO_EVENTS_EXTENDED` - Extended automatic user events +- `APPSEC_AUTO_EVENTS_RC` - User ID collection via RC + +Other scenarios: +- `GRAPHQL_APPSEC` - AppSec for GraphQL +- `APPSEC_LOW_WAF_TIMEOUT` - WAF with low timeout +- `APPSEC_RULES_MONITORING_WITH_ERRORS` - Rules with errors +- `EVERYTHING_DISABLED` - AppSec disabled tests + +Scenario groups (run all scenarios in a group): +- `APPSEC_SCENARIOS` - Most AppSec scenarios +- `APPSEC_RASP_SCENARIOS` - RASP-specific tests +- `REMOTE_CONFIG_SCENARIOS` - Remote configuration tests +- `ESSENTIALS_SCENARIOS` - Essential/core tests + +### Verifying Which Helper is Running + +After running a test, check `logs_/docker/weblog/logs/helper.log` to determine which helper was used: + +**C++ helper** log messages: +``` +[timestamp][info][pid] Sending log messages to binding, min level info +[timestamp][info][pid] Started listening on abstract socket: @/ddappsec/... +[timestamp][info][pid] starting runner on new thread +[timestamp][info][pid] Runner running +[timestamp][info][pid] DDAS-0014-00: AppSec has started +``` + +**Rust helper** log messages: +``` +2026-01-15T19:41:20.269325053Z [INFO] AppSec helper starting +2026-01-15T19:41:20.269907428Z [INFO] Configuration: Config { socket_path: ... +2026-01-15T19:41:20.277712636Z [INFO] AppSec helper started successfully +2026-01-15T19:41:20.277760178Z [INFO] Starting server on socket: ... +2026-01-15T19:41:20.277871261Z [INFO] Listening for connections +``` + +Key differences: +- C++ uses `[timestamp][level][pid]` format and messages like "Runner running", "starting runner on new thread" +- Rust uses `ISO8601_TIMESTAMP [LEVEL]` format and messages like "AppSec helper starting", "Listening for connections" + +## GitLab CI + +The dd-trace-php repository is mirrored from GitHub to GitLab at `DataDog/apm-reliability/dd-trace-php` via gitsync. CI pipelines run on the GitLab mirror. + +### Pipeline Structure + +The main `.gitlab-ci.yml` uses PHP generators to create child pipelines: + +1. **Parent pipeline** runs `generate-templates` job which executes PHP scripts: + - `.gitlab/generate-appsec.php` → `appsec-gen.yml` + - `.gitlab/generate-tracer.php` → `tracer-gen.yml` + - `.gitlab/generate-profiler.php` → `profiler-gen.yml` + - `.gitlab/generate-package.php` → `package-gen.yml` + - `.gitlab/generate-shared.php` → `shared-gen.yml` + +2. **Child pipelines** are triggered from the generated YAML artifacts + +### Helper-Rust CI Jobs + +The appsec child pipeline (generated from `generate-appsec.php`) includes these helper-rust jobs: + +| Job | Description | +|-----|-------------| +| `helper-rust build and test` | Builds helper-rust and runs `cargo test` + format check | +| `helper-rust code coverage` | Runs unit tests with coverage, uploads to codecov | +| `helper-rust integration coverage` | Runs integration tests with coverage-instrumented binary | +| `appsec integration tests (helper-rust)` | Integration tests using the Rust helper (PHP 7.4, 8.1, 8.3, 8.4-zts) | + +### Checking Pipeline Status + +The appsec child pipeline IID can be found in the parent pipeline's downstream pipelines. Key jobs to monitor: +- Jobs with `helper-rust` in the name for Rust-specific CI +- Jobs with `(helper-rust)` suffix run integration tests with the Rust implementation + +### Reading Job Logs + +The GitLab MCP tools don't include a job trace/log reader. To read job logs via the API: + +```bash +# Extract the token from MCP config (project ID 355 = DataDog/apm-reliability/dd-trace-php) +jq -r '.mcpServers.gitlab.env.GITLAB_PERSONAL_ACCESS_TOKEN' ~/.claude.json + +# Then fetch job trace using the token +curl -s -H "PRIVATE-TOKEN: " "https://gitlab.ddbuild.io/api/v4/projects/355/jobs//trace" > /tmp/... +``` + +### Monitoring CI Jobs + +Use the helper script to check job status: + +```bash +# Check helper-rust jobs in a pipeline (use numeric pipeline ID, not IID) +./scripts/check-ci-jobs.sh helper-rust +``` + +To monitor a pipeline until completion and get notified, spawn a background agent with this prompt: + +``` +Monitor GitLab pipeline for helper-rust jobs. Run this loop: +1. Run: ./scripts/check-ci-jobs.sh helper-rust +2. Parse the output to get RUNNING, PASSED, FAILED counts +3. If FAILED > 0, use speak_when_done MCP to say "X helper rust jobs failed" and STOP +4. If RUNNING > 0: + - First iteration: wait 60 seconds + - Subsequent iterations: wait 300 seconds + - Then repeat from step 1 +5. When RUNNING == 0 and FAILED == 0, use speak_when_done MCP to say "All helper rust jobs passed" +``` + +## Misc Notes + +- Always ask for confirmation before reverting, committing, or pushing anything with git +- Do not run commands with simply tail, as that prevents checking the progress. If you're to use tail, use also tee /tmp/<logfile> diff --git a/appsec/helper-rust/Cargo.lock b/appsec/helper-rust/Cargo.lock new file mode 100644 index 00000000000..b0009773aa6 --- /dev/null +++ b/appsec/helper-rust/Cargo.lock @@ -0,0 +1,2414 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e0fee31ef5ed1ba1316088939cea399010ed7731dba877ed44aeb407a75ea" +dependencies = [ + "backtrace", +] + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-rs" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "cc" +version = "1.2.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +dependencies = [ + "cc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89e8918065695684b2b0702da20382d5ae6065cf3327bc2d6436bd49a71ce9f3" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "helper-rust" +version = "0.1.0" +dependencies = [ + "anyhow", + "arc-swap", + "base64", + "cc", + "crossbeam-epoch", + "flate2", + "futures", + "libc", + "libddwaf", + "libddwaf-sys", + "log", + "rmp-serde", + "serde", + "serde_json", + "serde_tuple", + "serial_test", + "simplelog", + "stderrlog", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.178" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" + +[[package]] +name = "libddwaf" +version = "2.0.0-alpha0" +dependencies = [ + "libddwaf-sys", + "serde", +] + +[[package]] +name = "libddwaf-sys" +version = "2.0.0-alpha0" +dependencies = [ + "bindgen", + "flate2", + "hyper-rustls", + "libc", + "reqwest", + "rustls", + "tar", + "zstd", +] + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libredox" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +dependencies = [ + "bitflags", + "libc", + "redox_syscall 0.6.0", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "litemap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +dependencies = [ + "value-bag", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec96166dafa0886eb81fe1c0a388bece180fbef2135f97c1e2cf8302e74b43b5" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "reqwest" +version = "0.12.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" +dependencies = [ + "byteorder", + "num-traits", + "paste", +] + +[[package]] +name = "rmp-serde" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" +dependencies = [ + "byteorder", + "rmp", + "serde", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_fmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e497af288b3b95d067a23a4f749f2861121ffcb2f6d8379310dcda040c345ed" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "serde_tuple" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af196b9c06f0aa5555ab980c01a2527b0f67517da8d68b1731b9d4764846a6f" +dependencies = [ + "serde", + "serde_tuple_macros", +] + +[[package]] +name = "serde_tuple_macros" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3a1e7d2eadec84deabd46ae061bf480a91a6bce74d25dad375bd656f2e19d8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serial_test" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d0b343e184fc3b7bb44dff0705fffcf4b3756ba6aff420dddd8b24ca145e555" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f50427f258fb77356e4cd4aa0e87e2bd2c66dbcee41dc405282cae2bfc26c83" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +dependencies = [ + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simplelog" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16257adbfaef1ee58b1363bdc0664c9b8e1e30aed86049635fb5f147d065a9c0" +dependencies = [ + "log", + "termcolor", + "time", +] + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stderrlog" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c910772f992ab17d32d6760e167d2353f4130ed50e796752689556af07dc6b" +dependencies = [ + "chrono", + "is-terminal", + "log", + "termcolor", + "thread_local", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "sval" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502b8906c4736190684646827fbab1e954357dfe541013bbd7994d033d53a1ca" + +[[package]] +name = "sval_buffer" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4b854348b15b6c441bdd27ce9053569b016a0723eab2d015b1fd8e6abe4f708" +dependencies = [ + "sval", + "sval_ref", +] + +[[package]] +name = "sval_dynamic" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bd9e8b74410ddad37c6962587c5f9801a2caadba9e11f3f916ee3f31ae4a1f" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_fmt" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe17b8deb33a9441280b4266c2d257e166bafbaea6e66b4b34ca139c91766d9" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_json" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854addb048a5bafb1f496c98e0ab5b9b581c3843f03ca07c034ae110d3b7c623" +dependencies = [ + "itoa", + "ryu", + "sval", +] + +[[package]] +name = "sval_nested" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96cf068f482108ff44ae8013477cb047a1665d5f1a635ad7cf79582c1845dce9" +dependencies = [ + "sval", + "sval_buffer", + "sval_ref", +] + +[[package]] +name = "sval_ref" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed02126365ffe5ab8faa0abd9be54fbe68d03d607cd623725b0a71541f8aaa6f" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_serde" +version = "2.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a263383c6aa2076c4ef6011d3bae1b356edf6ea2613e3d8e8ebaa7b57dd707d5" +dependencies = [ + "serde_core", + "sval", + "sval_nested", +] + +[[package]] +name = "syn" +version = "2.0.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "termcolor" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "value-bag" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16530907bfe2999a1773ca5900a65101e092c70f642f25cc23ca0c43573262c5" +dependencies = [ + "erased-serde", + "serde_core", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00ae130edd690eaa877e4f40605d534790d1cf1d651e7685bd6a144521b251f" +dependencies = [ + "sval", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "writeable" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/appsec/helper-rust/Cargo.toml b/appsec/helper-rust/Cargo.toml new file mode 100644 index 00000000000..fd48ca0e985 --- /dev/null +++ b/appsec/helper-rust/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "helper-rust" +version = "0.1.0" +edition = "2021" + +[workspace] + +[lib] +name = "ddappsec_helper_rust" +path = "src/lib.rs" +crate-type = ["cdylib"] + +[dependencies] +# Tokio runtime with all features enabled +tokio = { version = "1.0", features = ["full"] } + +serde = { version = "1.0", features = ["derive"] } +serde_tuple = { version = "1.0" } +rmp-serde = "1.1" # MessagePack support for serde +serde_json = "1.0" +anyhow = { version = "1.0", features = ["backtrace"] } +thiserror = "1.0" +futures = "0.3" +log = { version = "0.4", features = ["kv", "kv_std"] } # Logging facade with key-value support +simplelog = "0.12" # Simple logger with file support +tokio-util = { version = "0.7", features = ["codec"] } +tokio-stream = "0.1" +stderrlog = "0.6.0" +libc = "0.2" +arc-swap = "1.7" +crossbeam-epoch = "0.9" # Epoch-based memory reclamation +libddwaf = { path = "../third_party/libddwaf-rust/crates/libddwaf" } +libddwaf-sys = { path = "../third_party/libddwaf-rust/crates/libddwaf-sys", default-features = false } +base64 = "0.22.1" +flate2 = "1.0" # gzip compression for schemas + +[features] +coverage = [] + +[build-dependencies] +cc = "1.0" + +[dev-dependencies] +serial_test = "3" + +[profile.dev] +debug = 2 diff --git a/appsec/helper-rust/Makefile b/appsec/helper-rust/Makefile new file mode 100644 index 00000000000..67e1a3f96dd --- /dev/null +++ b/appsec/helper-rust/Makefile @@ -0,0 +1,45 @@ +MUSL_GCC_SYSROOT_AMD64 := $(realpath $(shell /opt/homebrew/bin/x86_64-linux-musl-gcc -v -E - < /dev/null 2>&1 | grep -o -- '-isysroot [^ ]*'| awk '{print $$2}')) +$(info MUSL_GCC_SYSROOT_AMD64 is '$(MUSL_GCC_SYSROOT_AMD64)') + +# Non-phony targets for libunwind downloads +target/libunwind-x86_64/libunwind.a target/libunwind-x86_64/liblzma.a: + mkdir -p target/libunwind-x86_64 + docker run --rm --platform linux/amd64 alpine:3.21 sh -c \ + "apk add --no-cache llvm-libunwind-static xz-static > /dev/null 2>&1 && cat /usr/lib/libunwind.a" \ + > target/libunwind-x86_64/libunwind.a + docker run --rm --platform linux/amd64 alpine:3.21 sh -c \ + "apk add --no-cache llvm-libunwind-static xz-static > /dev/null 2>&1 && cat /usr/lib/liblzma.a" \ + > target/libunwind-x86_64/liblzma.a + +bcross_linux_amd64: target/libunwind-x86_64/libunwind.a target/libunwind-x86_64/liblzma.a + RUSTC_LOG=rustc_codegen_ssa::back::link=debug \ + BINDGEN_EXTRA_CLANG_ARGS="--sysroot=$(MUSL_GCC_SYSROOT_AMD64) --target=x86_64-linux-musl" \ + AR=/opt/homebrew/opt/binutils/bin/ar \ + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS='-C linker=/opt/homebrew/bin/x86_64-linux-musl-gcc -C target-feature=-crt-static -L $(MUSL_GCC_SYSROOT_AMD64)/lib -L target/libunwind-x86_64' \ + cargo +nightly build \ + --target x86_64-unknown-linux-musl + /opt/homebrew/bin/patchelf --remove-needed libc.so target/x86_64-unknown-linux-musl/debug/libddappsec_helper_rust.so +.PHONY: bcross_linux_amd64 + +MUSL_GCC_SYSROOT := $(realpath $(shell /opt/homebrew/bin/aarch64-linux-musl-gcc -v -E - < /dev/null 2>&1 | grep -o -- '-isysroot [^ ]*'| awk '{print $$2}')) +$(info MUSL_GCC_SYSROOT is '$(MUSL_GCC_SYSROOT)') + +# Non-phony targets for libunwind downloads +target/libunwind-aarch64/libunwind.a target/libunwind-aarch64/liblzma.a: + mkdir -p target/libunwind-aarch64 + docker run --rm --platform linux/arm64 alpine:3.21 sh -c \ + "apk add --no-cache llvm-libunwind-static xz-static > /dev/null 2>&1 && cat /usr/lib/libunwind.a" \ + > target/libunwind-aarch64/libunwind.a + docker run --rm --platform linux/arm64 alpine:3.21 sh -c \ + "apk add --no-cache llvm-libunwind-static xz-static > /dev/null 2>&1 && cat /usr/lib/liblzma.a" \ + > target/libunwind-aarch64/liblzma.a + +bcross_linux_aarch64: target/libunwind-aarch64/libunwind.a target/libunwind-aarch64/liblzma.a + RUSTC_LOG=rustc_codegen_ssa::back::link=debug \ + BINDGEN_EXTRA_CLANG_ARGS="--sysroot=$(MUSL_GCC_SYSROOT) --target=aarch64-linux-musl" \ + AR=/opt/homebrew/opt/binutils/bin/ar \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS='-C linker=/opt/homebrew/bin/aarch64-linux-musl-gcc -L $(MUSL_GCC_SYSROOT)/lib -L target/libunwind-aarch64' \ + cargo +nightly build --target aarch64-unknown-linux-musl + /opt/homebrew/bin/patchelf --remove-needed libc.so target/aarch64-unknown-linux-musl/debug/libddappsec_helper_rust.so +.PHONY: bcross_linux_aarch64 + diff --git a/appsec/helper-rust/build.rs b/appsec/helper-rust/build.rs new file mode 100644 index 00000000000..d849ee15aeb --- /dev/null +++ b/appsec/helper-rust/build.rs @@ -0,0 +1,98 @@ +use std::env; +use std::path::PathBuf; + +fn main() { + let target = env::var("TARGET").expect("TARGET environment variable not set"); + let has_coverage = env::var("CARGO_FEATURE_COVERAGE").is_ok(); + + // Compile and link glibc compatibility shim for musl targets + if target.contains("musl") { + cc::Build::new() + .file("glibc_compat.c") + .compile("glibc_compat"); + + println!("cargo::rerun-if-changed=glibc_compat.c"); + } + + // When building with coverage instrumentation, compile coverage initialization code + // that configures LLVM profiling runtime at library load time + if has_coverage { + let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set"); + + cc::Build::new() + .file("coverage_init.c") + .define("COVERAGE_BUILD", None) + .compile("coverage_init"); + + // On aarch64, the LLVM profiler runtime requires outline-atomic functions + // that aren't available in older glibc versions + let needs_outline_atomics = target.contains("aarch64"); + if needs_outline_atomics { + cc::Build::new() + .file("outline_atomics.c") + .compile("outline_atomics"); + println!("cargo::rerun-if-changed=outline_atomics.c"); + } + + // Force the linker to include coverage_init.a and outline_atomics.a in their + // entirety, even though nothing in Rust references their symbols directly. + // coverage_init.c has constructor/destructor functions for flushing coverage. + // outline_atomics.c provides atomic helpers needed by the profiler runtime. + println!("cargo::rustc-link-arg=-Wl,--whole-archive"); + println!("cargo::rustc-link-arg={}/libcoverage_init.a", out_dir); + if needs_outline_atomics { + println!("cargo::rustc-link-arg={}/liboutline_atomics.a", out_dir); + } + println!("cargo::rustc-link-arg=-Wl,--no-whole-archive"); + + println!("cargo::rerun-if-changed=coverage_init.c"); + } + + // Add $ORIGIN (Linux) or @loader_path (macOS) to allow finding libraries + // in the same directory as the binary/library + if target.contains("linux") { + println!("cargo::rustc-link-arg=-Wl,-rpath,$ORIGIN"); + } else if target.contains("darwin") || target.contains("apple") { + println!("cargo::rustc-link-arg=-Wl,-rpath,@loader_path"); + } + + // If LIBDDWAF_PREFIX is set, add that library path to rpath as well + // This matches the behavior in libddwaf-sys build.rs + if let Ok(prefix) = env::var("LIBDDWAF_PREFIX") { + let lib_dir = PathBuf::from(prefix).join("lib"); + println!("cargo::rustc-link-arg=-Wl,-rpath,{}", lib_dir.display()); + } + + println!("cargo::rerun-if-env-changed=LIBDDWAF_PREFIX"); + + build_test_sidecar_lib(); +} + +fn build_test_sidecar_lib() { + let out_dir = env::var("OUT_DIR").expect("OUT_DIR not set"); + let target = env::var("TARGET").expect("TARGET not set"); + let host = env::var("HOST").expect("HOST not set"); + + let (lib_name, shared_flag) = if target.contains("darwin") || target.contains("apple") { + ("libtest_sidecar.dylib", "-dynamiclib") + } else { + ("libtest_sidecar.so", "-shared") + }; + + let src = "test_sidecar_lib.c"; + let out = PathBuf::from(&out_dir).join(lib_name); + + let status = cc::Build::new() + .target(&target) + .host(&host) + .get_compiler() + .to_command() + .args([shared_flag, "-fPIC", "-o"]) + .arg(&out) + .arg(src) + .status() + .expect("Failed to run C compiler for test fixture"); + + assert!(status.success(), "Failed to compile test sidecar library"); + println!("cargo::rerun-if-changed={}", src); +} diff --git a/appsec/helper-rust/coverage_init.c b/appsec/helper-rust/coverage_init.c new file mode 100644 index 00000000000..17077b3e02c --- /dev/null +++ b/appsec/helper-rust/coverage_init.c @@ -0,0 +1,126 @@ +// LLVM coverage profiling initialization for helper-rust +// +// This module sets up the LLVM profiling environment when the helper is +// loaded by sidecar. It reinitializes the LLVM profiling runtime to use +// the correct output path. +// +// Coverage data is flushed when the process receives SIGUSR1, as well as +// at normal process exit via atexit handler. + +#ifdef COVERAGE_BUILD + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// LLVM profiling runtime API (from compiler-rt/lib/profile/) +extern void __llvm_profile_initialize_file(void); +extern int __llvm_profile_write_file(void); +extern const char *__llvm_profile_get_filename(void); + +static int log_fd = -1; + +static void coverage_log(const char *fmt, ...) { + if (log_fd < 0) { + log_fd = open("/helper-rust/coverage/coverage_init.log", + O_WRONLY | O_CREAT | O_APPEND, 0644); + if (log_fd < 0) { + log_fd = STDERR_FILENO; + } + } + + char buf[1024]; + int offset = 0; + + time_t now = time(NULL); + struct tm *tm_info = localtime(&now); + offset += strftime(buf, sizeof(buf), "[%Y-%m-%d %H:%M:%S] ", tm_info); + offset += snprintf(buf + offset, sizeof(buf) - offset, "[pid=%d] ", getpid()); + + va_list args; + va_start(args, fmt); + offset += vsnprintf(buf + offset, sizeof(buf) - offset, fmt, args); + va_end(args); + + if (offset < (int)sizeof(buf) - 1) { + buf[offset++] = '\n'; + } + + write(log_fd, buf, offset); +} + +static void coverage_flush(void) { + coverage_log("coverage_flush() called, flushing coverage data"); + + int ret = __llvm_profile_write_file(); + coverage_log("__llvm_profile_write_file() returned %d (errno=%d: %s)", + ret, errno, strerror(errno)); + + const char *filename = __llvm_profile_get_filename(); + coverage_log("Final profile filename: %s", filename ? filename : "(null)"); +} + +static void coverage_atexit(void) { + coverage_log("coverage_atexit() called"); + coverage_flush(); + if (log_fd >= 0 && log_fd != STDERR_FILENO) { + close(log_fd); + log_fd = -1; + } +} + +static void coverage_signal_handler(int signo) { + (void)signo; + coverage_log("SIGUSR1 received, flushing coverage data"); + int ret = __llvm_profile_write_file(); + coverage_log("__llvm_profile_write_file() returned %d", ret); +} + +__attribute__((constructor)) +static void coverage_init(void) { + coverage_log("coverage_init() constructor called"); + + const char *env_profile = getenv("LLVM_PROFILE_FILE"); + coverage_log("LLVM_PROFILE_FILE env = %s", env_profile ? env_profile : "(null)"); + + const char *filename_before = __llvm_profile_get_filename(); + coverage_log("Profile filename BEFORE reinit: %s", + filename_before ? filename_before : "(null)"); + + const char *profile_path = "/helper-rust/coverage/helper-%p-%m.profraw"; + coverage_log("Setting LLVM_PROFILE_FILE to: %s", profile_path); + setenv("LLVM_PROFILE_FILE", profile_path, 1); + + coverage_log("Calling __llvm_profile_initialize_file()"); + __llvm_profile_initialize_file(); + + const char *filename_after = __llvm_profile_get_filename(); + coverage_log("Profile filename AFTER reinit: %s", + filename_after ? filename_after : "(null)"); + + signal(SIGUSR1, coverage_signal_handler); + coverage_log("Signal handler installed for SIGUSR1"); + + atexit(coverage_atexit); + coverage_log("Registered atexit handler"); + + coverage_log("coverage_init() completed"); +} + +__attribute__((destructor)) +static void coverage_fini(void) { + coverage_log("coverage_fini() destructor called"); + coverage_flush(); + if (log_fd >= 0 && log_fd != STDERR_FILENO) { + close(log_fd); + log_fd = -1; + } +} + +#endif // COVERAGE_BUILD diff --git a/appsec/helper-rust/glibc_compat.c b/appsec/helper-rust/glibc_compat.c new file mode 100644 index 00000000000..75cfb25b158 --- /dev/null +++ b/appsec/helper-rust/glibc_compat.c @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include + +#if defined(__linux__) && !defined(__GLIBC__) + +# ifdef __x86_64__ +float ceilf(float x) +{ + float result; + // NOLINTNEXTLINE(hicpp-no-assembler) + __asm__("roundss $0x0A, %[x], %[result]" + : [result] "=x"(result) + : [x] "x"(x)); + return result; +} +double ceil(double x) +{ + double result; + // NOLINTNEXTLINE(hicpp-no-assembler) + __asm__("roundsd $0x0A, %[x], %[result]" + : [result] "=x"(result) + : [x] "x"(x)); + return result; +} +# endif + +# ifdef __aarch64__ +float ceilf(float x) +{ + float result; + __asm__("frintp %s0, %s1\n" : "=w"(result) : "w"(x)); + return result; +} +double ceil(double x) +{ + double result; + __asm__("frintp %d0, %d1\n" : "=w"(result) : "w"(x)); + return result; +} +# endif + +# ifdef __aarch64__ +# define _STAT_VER 0 +# else +# define _STAT_VER 1 +# endif + +// glibc before 2.33 (2021) doesn't have these +int stat(const char *restrict path, void *restrict buf) +{ + int __xstat(int, const char *restrict, void *restrict); + return __xstat(_STAT_VER, path, buf); +} + +int fstat(int fd, void *buf) +{ + int __fxstat(int, int, void *); + return __fxstat(_STAT_VER, fd, buf); +} + +int lstat(const char *restrict path, void *restrict buf) +{ + int __lxstat(int, const char *restrict, void *restrict); + return __lxstat(_STAT_VER, path, buf); +} + +int fstatat(int dirfd, const char *restrict pathname, void *restrict statbuf, int flags) +{ + int __fxstatat(int, int, const char *restrict, void *restrict, int); + return __fxstatat(_STAT_VER, dirfd, pathname, statbuf, flags); +} + +// glibc doesn't define pthread_atfork on aarch64. We need to delegate to +// glibc's __register_atfork() instead. __register_atfork() takes an extra +// argument, __dso_handle, which is a pointer to the DSO that is registering the +// fork handlers. This is used to ensure that the handlers are not called after +// the DSO is unloaded. glibc on amd64 also implements pthread_atfork() in terms +// of __register_atfork(). (musl never unloads modules so that potential +// problem doesn't exist) + +// On amd64, even though pthread_atfork is exported by glibc, it should not be +// used. Code that uses pthread_atfork will compile to an import to +// __register_atfork(), but here we're compiling against musl, resulting in an +// an import to pthread_atfork. This will cause a runtime error after the test +// that unloads our module. The reason is that when we call pthread_atfork in +// glibc, __register_atfork() is called with the __dso_handle of libc6.so, not +// the __dso_handle of our module. So the fork handler is not unregistered when +// our module is unloaded. + +extern void *__dso_handle __attribute__((weak)); +int __register_atfork(void (*prepare)(void), void (*parent)(void), + void (*child)(void), void *__dso_handle) __attribute__((weak)); + +int pthread_atfork( + void (*prepare)(void), void (*parent)(void), void (*child)(void)) +{ + // glibc + if (__dso_handle && __register_atfork) { + return __register_atfork(prepare, parent, child, __dso_handle); + } + + static int (*real_atfork)(void (*)(void), void (*)(void), void (*)(void)); + + if (!real_atfork) { + // dlopen musl +# ifdef __aarch64__ + void *handle = dlopen("ld-musl-aarch64.so.1", RTLD_LAZY); + if (!handle) { + (void)fprintf( + // NOLINTNEXTLINE(concurrency-mt-unsafe) + stderr, "dlopen of ld-musl-aarch64.so.1 failed: %s\n", + dlerror()); + abort(); + } +# else + void *handle = dlopen("libc.musl-x86_64.so.1", RTLD_LAZY); + if (!handle) { + (void)fprintf( + // NOLINTNEXTLINE(concurrency-mt-unsafe) + stderr, "dlopen of libc.musl-x86_64.so.1 failed: %s\n", + dlerror()); + abort(); + } +# endif + real_atfork = dlsym(handle, "pthread_atfork"); + if (!real_atfork) { + (void)fprintf( + // NOLINTNEXTLINE(concurrency-mt-unsafe) + stderr, "dlsym of pthread_atfork failed: %s\n", dlerror()); + abort(); + } + } + + return real_atfork(prepare, parent, child); +} + +# ifdef __x86_64__ +struct pthread_cond; +struct pthread_condattr; +typedef struct pthread_cond pthread_cond_t; +typedef struct pthread_condattr pthread_condattr_t; + +int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr) +{ + static int (*real_pthread_cond_init)(pthread_cond_t *cond, const pthread_condattr_t *cond_attr); + + if (!real_pthread_cond_init) { + void *handle = dlopen("libc.so.6", RTLD_LAZY); + if (!handle) { + void *handle = dlopen("libc.musl-x86_64.so.1", RTLD_LAZY); + if (!handle) { + (void)fprintf( + // NOLINTNEXTLINE(concurrency-mt-unsafe) + stderr, "dlopen of libc.so.6 and libc.musl-x86_64.so.1 failed: %s\n", + dlerror()); + abort(); + } + } + + real_pthread_cond_init = dlsym(handle, "pthread_cond_init"); + if (!real_pthread_cond_init) { + (void)fprintf( + // NOLINTNEXTLINE(concurrency-mt-unsafe) + stderr, "dlsym of pthread_cond_init failed: %s\n", dlerror()); + abort(); + } + } + + return real_pthread_cond_init(cond, cond_attr); +} +# endif + +// the symbol strerror_r in glibc is not the POSIX version; it returns char * +// __xpg_sterror_r is exported by both glibc and musl +int strerror_r(int errnum, char *buf, size_t buflen) +{ + int __xpg_strerror_r(int, char *, size_t); + return __xpg_strerror_r(errnum, buf, buflen); +} + +// when compiling with --coverage, some references to atexit show up. +// glibc doesn't provide atexit for similar reasons as pthread_atfork presumably +int __cxa_atexit(void (*func)(void *), void *arg, void *dso_handle); +int atexit(void (*function)(void)) +{ + if (!__dso_handle) { + (void)fprintf(stderr, "Aborting because __dso_handle is NULL\n"); + abort(); + } + + // the cast is harmless on amd64 and aarch64. Passing an extra argument to a + // function that expects none causes no problems + return __cxa_atexit((void (*)(void *))function, 0, __dso_handle); +} + +// introduced in glibc 2.25 +ssize_t getrandom(void *buf, size_t buflen, unsigned int flags) { + (void)flags; + // SYS_getrandom is 318 (amd64) or 278 (aarch64) + // This was only added in Linux 3.17 (2014), so don't use it + // return syscall(SYS_getrandom, buf, buflen, flags); + int fd; + size_t bytes_read = 0; + + fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + return -1; + } + + while (bytes_read < buflen) { + ssize_t result = read(fd, (char*)buf + bytes_read, buflen - bytes_read); + if (result < 0) { + if (errno == EINTR) { + continue; + } + close(fd); + return -1; + } + bytes_read += result; + } + + close(fd); + return (ssize_t)bytes_read; +} + +#ifdef __x86_64__ +#define MEMFD_CREATE_SYSCALL 319 +#elif __aarch64__ +#define MEMFD_CREATE_SYSCALL 279 +#endif + +// introduced in glibc 2.27 +int memfd_create(const char *name, unsigned flags) { + return syscall(MEMFD_CREATE_SYSCALL, name, flags); +} + +#endif diff --git a/appsec/helper-rust/outline_atomics.c b/appsec/helper-rust/outline_atomics.c new file mode 100644 index 00000000000..9c7d3ddecae --- /dev/null +++ b/appsec/helper-rust/outline_atomics.c @@ -0,0 +1,42 @@ +// Outline-atomic helpers for aarch64 +// These are needed when linking with Rust's libprofiler_builtins +// which was compiled with outline-atomics support. +// On older glibc/gcc versions, these functions are not provided. + +#if defined(__aarch64__) && defined(__linux__) + +#include + +// Compare-and-swap operation: atomically compare *ptr with expected, +// and if equal, replace with desired. Returns the previous value of *ptr. +uint64_t __aarch64_cas8_sync(uint64_t expected, uint64_t desired, uint64_t *ptr) { + uint64_t prev; + int tmp; + __asm__ __volatile__( + "1: ldaxr %0, [%3]\n" // Load-acquire exclusive + " cmp %0, %1\n" // Compare with expected + " b.ne 2f\n" // Branch if not equal + " stlxr %w2, %4, [%3]\n" // Store-release exclusive + " cbnz %w2, 1b\n" // Retry if store failed + "2:" + : "=&r" (prev), "+r" (expected), "=&r" (tmp) + : "r" (ptr), "r" (desired) + : "cc", "memory"); + return prev; +} + +// Atomic add operation: atomically add value to *ptr and return the previous value +uint64_t __aarch64_ldadd8_sync(uint64_t value, uint64_t *ptr) { + uint64_t prev, tmp; + __asm__ __volatile__( + "1: ldaxr %0, [%2]\n" // Load-acquire exclusive + " add %1, %0, %3\n" // Add value + " stlxr %w1, %1, [%2]\n" // Store-release exclusive + " cbnz %w1, 1b\n" // Retry if store failed + : "=&r" (prev), "=&r" (tmp) + : "r" (ptr), "r" (value) + : "cc", "memory"); + return prev; +} + +#endif diff --git a/appsec/helper-rust/scripts/check-ci-jobs.sh b/appsec/helper-rust/scripts/check-ci-jobs.sh new file mode 100755 index 00000000000..2cd03cfc30d --- /dev/null +++ b/appsec/helper-rust/scripts/check-ci-jobs.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Check status of helper-rust CI jobs in a GitLab pipeline +# Usage: check-ci-jobs.sh [job_filter] +# Example: check-ci-jobs.sh 91141215 helper-rust +# Note: Use the numeric pipeline ID (not IID) from the child appsec pipeline + +set -e + +PIPELINE_ID="${1:?Pipeline ID required}" +JOB_FILTER="${2:-helper-rust}" +PROJECT_ID="355" +GITLAB_URL="https://gitlab.ddbuild.io" + +# Get token from MCP config +GITLAB_TOKEN=$(jq -r '.mcpServers.gitlab.env.GITLAB_PERSONAL_ACCESS_TOKEN' ~/.claude.json) + +if [ -z "$GITLAB_TOKEN" ] || [ "$GITLAB_TOKEN" = "null" ]; then + echo "ERROR: Could not extract GitLab token from ~/.claude.json" + exit 1 +fi + +# Get jobs matching filter +JOBS=$(curl -s -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \ + "$GITLAB_URL/api/v4/projects/$PROJECT_ID/pipelines/$PIPELINE_ID/jobs?per_page=100" | \ + jq -r ".[] | select(.name | contains(\"$JOB_FILTER\"))") + +if [ -z "$JOBS" ] || [ "$JOBS" = "" ]; then + echo "ERROR: No jobs found matching filter '$JOB_FILTER' in pipeline $PIPELINE_ID" + exit 1 +fi + +# Count statuses +TOTAL=$(echo "$JOBS" | jq -s 'length') +RUNNING=$(echo "$JOBS" | jq -s '[.[] | select(.status == "running" or .status == "pending" or .status == "created")] | length') +PASSED=$(echo "$JOBS" | jq -s '[.[] | select(.status == "success")] | length') +FAILED=$(echo "$JOBS" | jq -s '[.[] | select(.status == "failed")] | length') + +# Output summary +echo "PIPELINE_ID=$PIPELINE_ID" +echo "TOTAL=$TOTAL" +echo "RUNNING=$RUNNING" +echo "PASSED=$PASSED" +echo "FAILED=$FAILED" + +# Output job details +echo "---JOBS---" +echo "$JOBS" | jq -r '[.name, .status, .id] | @tsv' diff --git a/appsec/helper-rust/scripts/generate-sidecar-ffi.sh b/appsec/helper-rust/scripts/generate-sidecar-ffi.sh new file mode 100755 index 00000000000..8a7f499da58 --- /dev/null +++ b/appsec/helper-rust/scripts/generate-sidecar-ffi.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# Generate sidecar FFI bindings using bindgen +# +# Usage: ./scripts/generate-sidecar-ffi.sh +# +# Requires: cargo install bindgen-cli + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +COMPONENTS_RS_DIR="$PROJECT_DIR/../../components-rs" +OUTPUT_FILE="$PROJECT_DIR/src/ffi/sidecar_ffi.rs" + +cd "$COMPONENTS_RS_DIR" + +bindgen sidecar.h \ + --allowlist-function 'ddog_sidecar_enqueue_telemetry_log' \ + --allowlist-function 'ddog_sidecar_enqueue_telemetry_point' \ + --allowlist-function 'ddog_sidecar_enqueue_telemetry_metric' \ + --allowlist-function 'ddog_sidecar_connect' \ + --allowlist-function 'ddog_sidecar_ping' \ + --allowlist-function 'ddog_sidecar_transport_drop' \ + --allowlist-function 'ddog_Error_drop' \ + --allowlist-function 'ddog_Error_message' \ + --allowlist-function 'ddog_MaybeError_drop' \ + --allowlist-type 'ddog_SidecarTransport' \ + --allowlist-type 'ddog_LogLevel' \ + --allowlist-type 'ddog_CharSlice' \ + --allowlist-type 'ddog_Slice_CChar' \ + --allowlist-type 'ddog_MaybeError' \ + --allowlist-type 'ddog_Option_Error' \ + --allowlist-type 'ddog_Option_Error_Tag' \ + --allowlist-type 'ddog_Error' \ + --allowlist-type 'ddog_Vec_U8' \ + --allowlist-type 'ddog_MetricNamespace' \ + --no-layout-tests \ + --no-doc-comments \ + --use-core \ + --raw-line '//! Auto-generated FFI bindings from components-rs/sidecar.h' \ + --raw-line '//!' \ + --raw-line '//! Regenerate with: ./scripts/generate-sidecar-ffi.sh' \ + --raw-line '//!' \ + --raw-line '//! Only includes types/functions needed for helper-rust as telemetry sender.' \ + --raw-line '#![allow(non_camel_case_types, non_upper_case_globals, dead_code)]' \ + --output "$OUTPUT_FILE" \ + -- -I. + +echo "Generated $OUTPUT_FILE" diff --git a/appsec/helper-rust/src/client.rs b/appsec/helper-rust/src/client.rs new file mode 100644 index 00000000000..c95cf806f5d --- /dev/null +++ b/appsec/helper-rust/src/client.rs @@ -0,0 +1,1329 @@ +use std::{ + cell::Cell, + collections::{hash_map::Entry, HashMap}, + io, + mem::MaybeUninit, + os::fd::{AsFd, AsRawFd}, + sync::{ + atomic::{self, AtomicU64}, + Arc, + }, + time::{Duration, Instant}, +}; + +use anyhow::{anyhow, Context}; +use futures::{future::Shared, SinkExt}; +use libddwaf::{object::WafObjectType, RunnableContext}; +use log::{debug, error, info, warning as warn}; +use protocol::{ClientInitResp, CommandResponse, ConfigFeaturesResp}; +use thiserror::Error; +use tokio::net::UnixStream; +use tokio_stream::StreamExt; +use tokio_util::sync::CancellationToken; + +use crate::{ + client::protocol::{RequestExecOptions, WafRunType}, + service::{Service, ServiceFixedConfig, ServiceManager}, + telemetry::{ + error_tel_ctx::{ + clear_error_telemetry_context, update_error_telemetry_context, + with_error_telemetry_handle, + }, + SidecarReadyFuture, SidecarStatus, SpanMetaGenerator, SpanMetaName, SpanMetricName, + SpanMetricsGenerator, SpanMetricsSubmitter, TelemetryLogsGenerator, + TelemetryMetricsGenerator, TelemetrySidecarLogSubmitter, TelemetrySidecarMetricSubmitter, + }, +}; + +mod attributes; +pub mod log; +mod metrics; +pub mod protocol; + +/// Smart pointer that tracks worker count for a service. +/// Increments on creation/clone, decrements on drop. +#[derive(Clone)] +struct TrackedService { + service: Arc, +} + +impl TrackedService { + fn new(service: Arc) -> Self { + service.increment_worker_count(); + Self { service } + } +} + +impl Drop for TrackedService { + fn drop(&mut self) { + self.service.decrement_worker_count(); + } +} + +impl std::ops::Deref for TrackedService { + type Target = Service; + fn deref(&self) -> &Self::Target { + &self.service + } +} + +pub struct Client { + pub id: u64, + service_manager: &'static ServiceManager, + service: Option, + sidecar_settings: Option, + metrics_last_registered: Cell>, +} + +static CLIENT_SERIAL: AtomicU64 = AtomicU64::new(0); +impl Client { + pub fn new(service_manager: &'static ServiceManager) -> Self { + Self { + id: CLIENT_SERIAL.fetch_add(1, atomic::Ordering::Relaxed), + service_manager, + service: None, + sidecar_settings: None, + metrics_last_registered: Default::default(), + } + } + + pub async fn entrypoint( + self, + stream: UnixStream, + sidecar_ready: Shared, + cancel_token: CancellationToken, + ) { + log::with_scoped_client_id( + self.id, + self.do_entrypoint(stream, sidecar_ready, cancel_token), + ) + .await; + } + + async fn do_entrypoint( + mut self, + stream: UnixStream, + sidecar_ready: Shared, + cancel_token: CancellationToken, + ) { + info!("starting"); + + let sidecar_ready = sidecar_ready.await; + if sidecar_ready != SidecarStatus::Ready { + info!("Sidecar is not ready, no telemetry will be submitted"); + return; + } + + let res = do_client_entrypoint(&mut self, stream, cancel_token).await; + match res { + Ok(_) => { + info!("ended normally"); + } + Err(err) if err.is::() => { + warn!("ended due to client connectivity issue: {}", err); + } + Err(err) => { + error!("ended with failure: {}", err); + } + } + } + + /// Can't be called before client_init + pub fn get_service(&self) -> &Service { + self.service.as_ref().expect("service not initialized") + } +} + +/// Indicates a clean client shutdown - the client properly closed its connection +/// after completing all pending writes. This is NOT an error condition. +#[derive(Debug, Error)] +#[error("Client closed connection cleanly")] +struct CleanShutdown; + +/// Indicates the client disconnected unexpectedly - partial data was received +/// before the connection was closed (client crash, kill, network issue). +/// This is a connectivity issue, not a protocol error. +#[derive(Debug, Error)] +#[error("Client disconnected with incomplete data: {0}")] +struct ForcefulDisconnect(io::Error); + +async fn do_client_entrypoint( + client: &mut Client, + stream: UnixStream, + cancel_token: CancellationToken, +) -> anyhow::Result<()> { + with_error_telemetry_handle(do_client_entrypoint_inner(client, stream, cancel_token)).await +} + +async fn do_client_entrypoint_inner( + client: &mut Client, + stream: UnixStream, + cancel_token: CancellationToken, +) -> anyhow::Result<()> { + check_peer_uid_unix(&stream).await?; + + let mut framed = tokio_util::codec::Framed::new(stream, protocol::CommandCodec); + + // first, client_init + match recv_command(&mut framed, &cancel_token).await { + Ok(protocol::Command::ClientInit(args)) => { + let resp = handle_client_init(client, *args); + match resp { + Ok(resp) => { + send_command_resp(&mut framed, resp).await?; + } + Err(err) => { + let cir = ClientInitResp { + version: protocol::VERSION_FOR_PROTO, + status: "fail".to_string(), + errors: vec![err.to_string()], + ..Default::default() + }; + send_command_resp(&mut framed, CommandResponse::ClientInit(cir)).await?; + return Err(err); + } + } + } + Ok(cmd) => { + return Err(anyhow!("expected client_init, got {:?}", cmd)); + } + Err(e) if e.is::() => { + info!("client closed connection before sending first message"); + return Ok(()); + } + Err(e) if e.is::() => { + warn!("client forcefully disconnected before sending first message"); + return Err(e); + } + Err(e) => { + return Err(e); + } + }; + + // then the request loop + loop { + match do_request_loop_iter(client, &mut framed, &cancel_token).await { + Ok(_) => { + debug!("request done; waiting for new one"); + } + Err(err) if err.is::() => { + return Ok(()); + } + Err(err) if err.is::() => { + info!("client disconnected unexpectedly"); + return Err(err); + } + Err(err) => { + error!("error in request loop: {}", err); + return Err(err); + } + } + } +} + +fn handle_client_init( + client: &mut Client, + args: protocol::ClientInitArgs, +) -> anyhow::Result> { + let telemetry_settings = args.telemetry_settings.clone(); + let sd = ServiceFixedConfig::new( + args.appsec_enabled == Some(true), + args.waf_config.clone(), + args.remote_config.clone(), + args.telemetry_settings, + ); + + client.sidecar_settings = Some(args.sidecar_settings.clone()); + + update_error_telemetry_context(args.sidecar_settings.clone(), telemetry_settings.clone()); + + let last_registration_time = &client.metrics_last_registered; + let mut tel_metric_submitter = TelemetrySidecarMetricSubmitter::create( + &args.sidecar_settings, + &telemetry_settings, + last_registration_time, + ); + + let service = client + .service_manager + .get_service(&sd, tel_metric_submitter.as_mut()); + + let mut cir = ClientInitResp { + version: protocol::VERSION_FOR_PROTO, + ..Default::default() + }; + + match service { + Ok(service) => { + if let Some(diag) = service.take_pending_init_diagnostics_legacy() { + diag.generate_span_metrics(&mut cir) + } + + cir.meta.insert( + crate::telemetry::WAF_VERSION.0.to_string(), + Service::waf_version().to_string(), + ); + cir.helper_runtime = Some("rust".to_string()); + + client.service = Some(TrackedService::new(service)); + cir.status = "ok".to_string(); + Ok(CommandResponse::ClientInit(cir)) + } + Err(err) => { + error!("client init handling error: {:?}", err); + Err(err).context("client init handling error") + } + } +} + +fn handle_config_sync(client: &mut Client, args: protocol::ConfigSyncArgs) { + let Some(ref service) = client.service else { + error!("ConfigSync received before client_init"); + return; + }; + + let cur_disc = service.fixed_config(); + let telemetry_settings = args.telemetry_settings.clone(); + let new_disc = cur_disc.new_from_config_sync(args); + + let new_disc = match new_disc { + None => { + debug!( + "Settings did not change after config_sync, still {:?}", + cur_disc.config_sync_settings() + ); + return; + } + Some(new_disc) => { + debug!( + "Settings changed after config_sync, {:?} -> {:?}", + cur_disc.config_sync_settings(), + new_disc.config_sync_settings() + ); + new_disc + } + }; + + if let Some(ref sidecar_settings) = client.sidecar_settings { + update_error_telemetry_context(sidecar_settings.clone(), telemetry_settings.clone()); + } else { + clear_error_telemetry_context(); + } + + let mut tel_metric_submitter = match client.sidecar_settings { + Some(ref sidecar_settings) => TelemetrySidecarMetricSubmitter::create( + sidecar_settings, + &telemetry_settings, + &client.metrics_last_registered, + ), + None => { + // this should have been set in client_init + error!("Cannot submit telemetry metrics: sidecar_settings unexpectadly not set"); + TelemetrySidecarMetricSubmitter::noop() + } + }; + + match client + .service_manager + .get_service(&new_disc, &mut *tel_metric_submitter) + { + Ok(new_service) => { + client.service = Some(TrackedService::new(new_service)); + } + Err(e) => { + error!("Failed to get service with new RC path: {}; will continue running with old service!", e); + } + } +} + +async fn do_request_loop_iter( + client: &mut Client, + framed: &mut tokio_util::codec::Framed, + cancel_token: &CancellationToken, +) -> anyhow::Result<()> { + // wait for any number of config_syncs, followed by request_init + let mut req_ctx = match recv_command(framed, cancel_token).await? { + protocol::Command::RequestInit(req) => { + let service = client.get_service(); + let config_snapshot = service.config_snapshot(); + + // if ASM is disabled, send ConfigFeatures(false) and we're done + if !config_snapshot.asm_enabled { + debug!("ASM disabled, sending config_features(enabled=false) to request_init"); + let resp = protocol::CommandResponse::ConfigFeatures(ConfigFeaturesResp { + enabled: false, + }); + send_command_resp(framed, resp).await?; + return Ok(()); + } + + let mut req_ctx = ReqContext::new(service, config_snapshot.clone()); + let result = req_ctx.run_waf(req.data, &protocol::RequestExecOptions::regular())?; + + let resp = protocol::CommandResponse::RequestInit(protocol::RequestInitResp { + triggers: &result.triggers, + actions: &result.actions, + force_keep: req_ctx.should_force_keep(service, result.waf_keep), + settings: req_ctx.settings(), + }); + send_command_resp(framed, resp).await?; + req_ctx + } + + protocol::Command::ConfigSync(args) => { + handle_config_sync(client, *args); + + let service = client.get_service(); + let enabled = service.config_snapshot().asm_enabled; + let resp = if enabled { + protocol::CommandResponse::ConfigFeatures(ConfigFeaturesResp { enabled: true }) + } else { + protocol::CommandResponse::ConfigSync + }; + send_command_resp(framed, resp).await?; + + submit_service_telemetry(client, service); + + return Ok(()); + } + + command => { + anyhow::bail!("unexpected command {:?}", command); + } + }; + + loop { + match recv_command(framed, cancel_token).await? { + protocol::Command::RequestExec(req) => { + let req_options = if req.options.subctx_id.is_some() { + &req.options + } else if has_server_request_address(&req.data) { + // allow overriding of server.request.* variables in request_exec + debug!( + "Running WAF on the main WAF context because \ + server.request.* variables are present: {:?}", + req.data + ); + &RequestExecOptions::regular() + } else { + // by default, run on a transient subcontext + &RequestExecOptions { + subctx_id: Some("request_exec_transient".into()), + subctx_last_call: true, + ..req.options + } + }; + + let result = req_ctx.run_waf(req.data, req_options)?; + + let resp = protocol::CommandResponse::RequestExec(protocol::RequestExecResp { + triggers: result.triggers, + actions: result.actions, + force_keep: req_ctx.should_force_keep(client.get_service(), result.waf_keep), + settings: HashMap::default(), + }); + send_command_resp(framed, resp).await?; + continue; + } + protocol::Command::RequestShutdown(req) => { + let data = if client + .get_service() + .should_extract_schema(req.api_sec_samp_key) + { + use libddwaf::waf_map; + let context_processor = waf_map! {("extract-schema", true)}; + + let old_len = req.data.len() as usize; + let mut new_data = libddwaf::object::WafMap::new((old_len + 1) as u16); + for (i, entry) in req.data.into_iter().enumerate() { + new_data[i] = entry; + } + new_data[old_len] = ("waf.context.processor", context_processor).into(); + new_data + } else { + req.data + }; + + let result = req_ctx.run_waf(data, &protocol::RequestExecOptions::regular())?; + + // span metrics / meta + let mut span_submitter = metrics::CollectingMetricsSubmitter::default(); + req_ctx.generate_span_metrics(&mut span_submitter); + req_ctx.generate_meta(&mut span_submitter); + + let service = client.get_service(); + let force_keep = req_ctx.should_force_keep(service, result.waf_keep); + + let resp = + protocol::CommandResponse::RequestShutdown(protocol::RequestShutdownResp { + triggers: result.triggers, + actions: result.actions, + force_keep, + settings: HashMap::default(), + meta: span_submitter.take_meta(), + metrics: span_submitter.take_metrics(), + }); + send_command_resp(framed, resp).await?; + + submit_context_telemetry_metrics(client, &mut req_ctx, req.input_truncated); + submit_service_telemetry(client, service); + + break; + } + command => { + anyhow::bail!("unexpected command {:?}", command); + } + } + } + + Ok(()) +} + +fn submit_service_telemetry(client: &Client, service: &Service) { + if let (Some(sidecar_settings), telemetry_settings) = + (&client.sidecar_settings, &service.telemetry_settings()) + { + debug!("Submitting service telemetry to sidecar"); + let mut submitter = + TelemetrySidecarLogSubmitter::create(sidecar_settings, telemetry_settings); + service.generate_telemetry_logs(&mut *submitter); + + let mut submitter = TelemetrySidecarMetricSubmitter::create( + sidecar_settings, + telemetry_settings, + &client.metrics_last_registered, + ); + service.generate_telemetry_metrics(&mut *submitter); + } else { + debug!( + "Cannot submit service telemetry: sidecar_settings={:?}, telemetry_settings={:?}", + client.sidecar_settings, + service.telemetry_settings() + ); + } +} + +fn submit_context_telemetry_metrics( + client: &Client, + req_ctx: &mut ReqContext, + input_truncated: bool, +) { + let Some(ref sidecar_settings) = client.sidecar_settings else { + warn!("Cannot submit context telemetry metrics: sidecar_settings not set"); + return; + }; + let service = client.get_service(); + let telemetry_settings = service.telemetry_settings(); + + let mut tel_metric_submitter = TelemetrySidecarMetricSubmitter::create( + sidecar_settings, + telemetry_settings, + &client.metrics_last_registered, + ); + + let waf_metrics = req_ctx.take_waf_metrics(input_truncated); + waf_metrics.generate_telemetry_metrics(&mut *tel_metric_submitter); +} + +struct WafRunResult { + triggers: Vec, + actions: Vec, + waf_keep: bool, +} + +struct ReqContext { + waf_ctx: libddwaf::Context, + waf_subctxs: HashMap, + config_snapshot: Arc, + limiter_result: Option, + waf_metrics: metrics::WafMetrics, + waf_attributes: attributes::CollectedWafAttributes, + waf_timeout: Duration, +} +impl ReqContext { + const DEFAULT_WAF_TIMEOUT: Duration = Duration::from_millis(50); + const MAX_PLAIN_SCHEMA_ALLOWED: usize = 260; + const MAX_SCHEMA_SIZE: usize = 25000; + + fn new(service: &Service, config_snapshot: Arc) -> Self { + let rules_version = config_snapshot.rules_version.clone(); + let waf_timeout = service + .configured_waf_timeout() + .unwrap_or(Self::DEFAULT_WAF_TIMEOUT); + + Self { + waf_ctx: service.new_context(), + waf_subctxs: HashMap::new(), + config_snapshot, + limiter_result: None, + waf_metrics: metrics::WafMetrics::new(rules_version), + waf_attributes: attributes::CollectedWafAttributes::new( + Self::MAX_PLAIN_SCHEMA_ALLOWED, + Self::MAX_SCHEMA_SIZE, + ), + waf_timeout, + } + } + + fn run_waf( + &mut self, + data: libddwaf::object::WafMap, + options: &protocol::RequestExecOptions, + ) -> anyhow::Result { + debug!("Running WAF with: {:?}, options: {:?}", data, options); + + let waf_timeout = self.waf_timeout; + let mut ctx = self.get_waf_runnable(options)?; + let maybe_res = tokio::task::block_in_place(|| ctx.run(data, waf_timeout)); + drop(ctx); + let res = match maybe_res { + Ok(res) => res, + Err(err) => { + self.waf_metrics.record_non_rasp_error_eval(); + anyhow::bail!("WAF evaluation error: {:?}", err); + } + }; + + debug!("WAF run result: {:?}", res); + let run_output = match res { + libddwaf::RunResult::Match(result) => result, + libddwaf::RunResult::NoMatch(result) => result, + }; + + let triggers = match run_output.events() { + Some(events) => convert_events_to_json(events.value())?, + None => Vec::new(), + }; + let actions = match run_output.actions() { + Some(actions) => convert_actions(actions.value(), !triggers.is_empty())?, + None => Vec::new(), + }; + let waf_keep = run_output.keep(); + + if let Some(attributes) = run_output.attributes() { + for attr_kv in attributes.value().iter() { + self.waf_attributes.add_attribute(attr_kv); + } + } + + match &options.run_type { + WafRunType::NonRasp => { + self.waf_metrics.record_non_rasp_eval(&run_output); + } + WafRunType::RaspRule(rule_type) => { + self.waf_metrics.record_rasp_eval(rule_type, &run_output); + } + } + + Ok(WafRunResult { + triggers, + actions, + waf_keep, + }) + } + + /// Depending on the options, return either the context or a (possibly new) + /// subcontext. The subcontext may be dropped after the return is dropped, + /// depending on the options.subctx_last_call flag. + fn get_waf_runnable( + &mut self, + options: &protocol::RequestExecOptions, + ) -> anyhow::Result { + enum RunnableCtx<'a> { + Borrowed(&'a mut dyn libddwaf::RunnableContext), + Owned(libddwaf::Subcontext), + } + impl<'a> libddwaf::RunnableContext for RunnableCtx<'a> { + fn run( + &mut self, + data: libddwaf::object::WafMap, + timeout: Duration, + ) -> Result { + match self { + RunnableCtx::Borrowed(ctx) => ctx.run(data, timeout), + RunnableCtx::Owned(ctx) => ctx.run(data, timeout), + } + } + } + match options.subctx_id.as_ref() { + None => Ok(RunnableCtx::Borrowed(&mut self.waf_ctx)), + Some(subctx_id) => { + if options.subctx_last_call { + let subctx = self + .waf_subctxs + .remove(subctx_id) + .or_else(|| self.waf_ctx.new_subcontext().ok()) // error should not happen + .ok_or(anyhow!("Failed to create subcontext"))?; + Ok(RunnableCtx::Owned(subctx)) + } else { + let waf_ctx = &mut self.waf_ctx; + let entry = self.waf_subctxs.entry(subctx_id.clone()); + match entry { + Entry::Occupied(entry) => Ok(RunnableCtx::Borrowed(entry.into_mut())), + Entry::Vacant(entry) => { + let subctx = entry.insert( + waf_ctx + .new_subcontext() + .map_err(|e| anyhow!("Failed to create subcontext: {}", e))?, + ); + Ok(RunnableCtx::Borrowed(subctx)) + } + } + } + } + } + } + + fn should_force_keep(&mut self, service: &Service, waf_keep: bool) -> bool { + // cache limiter result (called once per request) + let limiter_allows = match self.limiter_result { + Some(result) => result, + None => { + let result = service.should_force_keep(); + self.limiter_result = Some(result); + result + } + }; + + limiter_allows && waf_keep + } + + fn settings(&self) -> HashMap<&'static str, String> { + HashMap::from([( + "auto_user_instrum", + self.config_snapshot.auto_user_instrum.as_str().to_string(), + )]) + } + + pub fn take_waf_metrics(&mut self, input_truncated: bool) -> metrics::WafMetrics { + self.waf_metrics.set_input_truncated(input_truncated); + std::mem::take(&mut self.waf_metrics) + } +} + +impl crate::telemetry::SpanMetricsGenerator for ReqContext { + fn generate_span_metrics(&'_ self, submitter: &mut dyn crate::telemetry::SpanMetricsSubmitter) { + self.waf_metrics.generate_span_metrics(submitter); + self.waf_attributes.generate_span_metrics(submitter); + } +} + +impl crate::telemetry::SpanMetaGenerator for ReqContext { + fn generate_meta(&'_ self, submitter: &mut dyn crate::telemetry::SpanMetricsSubmitter) { + // EVENT_RULES_VERSION is sent on every request_shutdown + // (WAF_VERSION and EVENT_RULES_ERRORS are sent only on client_init) + if let Some(rules_ver) = self.config_snapshot.rules_version.as_deref() { + submitter.submit_meta(crate::telemetry::EVENT_RULES_VERSION, rules_ver.to_string()); + } + } +} + +fn convert_events_to_json(events: &libddwaf::object::WafArray) -> anyhow::Result> { + events + .iter() + .try_fold(Vec::new(), |mut acc, event| -> anyhow::Result<_> { + let event_str = serde_json::to_string(event)?; + acc.push(event_str); + Ok(acc) + }) +} + +// the extension expects different names in the protocol message +fn map_action_name(waf_action: &str) -> Option<&'static str> { + match waf_action { + "block_request" => Some("block"), + "redirect_request" => Some("redirect"), + "generate_stack" => Some("stack_trace"), + "generate_schema" => Some("extract_schema"), + // "monitor" is reserved but not used in the WAF + _ => None, + } +} + +// convert ddwaf map {: {: }} to Vec, +// with some massaging to make inject "record" action. +fn convert_actions( + actions: &libddwaf::object::WafMap, + has_triggers: bool, +) -> anyhow::Result> { + let conv_actions = actions + .iter() + .try_fold(Vec::new(), |mut acc, kv| -> anyhow::Result<_> { + let waf_action_name = kv.key_str().map_err(|e| anyhow!(e.to_string()))?; + let action = match map_action_name(waf_action_name) { + Some(mapped) => mapped, + None => { + warn!("Unknown WAF action type: {}", waf_action_name); + return Ok(acc); // skip unknown actions + } + }; + let parameters = kv + .value() + .as_type::() + .ok_or(anyhow!("Action parameter map not a map"))? + .iter() + .try_fold(HashMap::new(), |mut acc, kv| -> anyhow::Result<_> { + let key = kv.key_str().map_err(|e| anyhow!(e.to_string()))?; + let value = kv.value(); + let value_str: String = match value.object_type() { + WafObjectType::String => { + value.as_type::() + .expect("We just checked it was a string") + .as_str() + .with_context(|| "Action parameter value is not a UTF-8 String")? + .into() + } + WafObjectType::Unsigned => + value.as_type::() + .expect("We just checked it was an unsigned") + .value().to_string(), + _ => { + anyhow::bail!("Action parameter value is not a string or unsigned: got {:?}. Full actions: {:?}", kv.value(), actions) + } + }; + acc.insert(key.to_owned(), value_str); + Ok(acc) + })?; + acc.push(protocol::ActionInstance { action, parameters }); + Ok(acc) + }); + + match conv_actions { + Ok(mut conv_actions) => { + maybe_inject_record_action(&mut conv_actions, has_triggers); + Ok(conv_actions) + } + Err(e) => Err(e), + } +} + +/// Injects a "record" action if there are triggers but no actions or +/// if there is a stack_trace action with no block/redirect/record action. +/// The extension only saves triggers if there is a record action. +fn maybe_inject_record_action(actions: &mut Vec, has_triggers: bool) { + if actions.is_empty() && has_triggers { + actions.push(protocol::ActionInstance { + action: "record", + parameters: HashMap::default(), + }); + } + + let mut event_action = false; + let mut stack_trace = false; + + for action in &*actions { + match action.action { + "block" | "redirect" | "record" => { + event_action = true; + } + "stack_trace" => { + stack_trace = true; + } + _ => {} + } + } + + if !event_action && stack_trace { + // Stacktrace needs to send a record as well so Appsec event is generated + actions.push(protocol::ActionInstance { + action: "record", + parameters: HashMap::default(), + }); + } +} + +fn has_server_request_address(data: &libddwaf::object::WafMap) -> bool { + data.iter().any(|kv| { + kv.key_str() + .map(|k| k.starts_with("server.request.")) + .unwrap_or(false) + }) +} + +async fn recv_command( + framed: &mut tokio_util::codec::Framed, + cancel_token: &CancellationToken, +) -> anyhow::Result { + debug!("Waiting for command"); + + tokio::select! { + maybe_msg = framed.next() => { + match maybe_msg { + Some(Ok(msg)) => { + debug!("Received command: {:?}", msg); + Ok(msg) + } + Some(Err(err)) => { + if is_incomplete_stream_error(&err) { + Err(ForcefulDisconnect(err).into()) + } else { + // Protocol error: invalid header marker, bad msgpack, unknown command + error!("Protocol error receiving command: {}", err); + let _ = framed.send(CommandResponse::ProtocolError).await; + Err(err.into()) + } + } + None => { + // Stream ended cleanly - client closed connection properly + Err(CleanShutdown.into()) + } + } + } + + _ = cancel_token.cancelled() => { + info!("Cancellation during client recv"); + Err(CleanShutdown.into()) + } + } +} + +fn is_incomplete_stream_error(err: &io::Error) -> bool { + // tokio_util's FramedRead returns this specific error when EOF is reached + // with bytes still in the decode buffer + err.kind() == io::ErrorKind::Other && err.to_string().contains("bytes remaining on stream") +} + +async fn send_command_resp( + framed: &mut tokio_util::codec::Framed, + cmd: protocol::CommandResponse<'_>, +) -> anyhow::Result<()> { + debug!("Sending command: {:?}", cmd); + match framed.send(cmd).await { + Ok(_) => Ok(()), + Err(err) => { + error!("Error sending command: {}", err); + Err(err)? + } + } +} + +async fn check_peer_uid_unix(stream: &UnixStream) -> anyhow::Result<()> { + let our_euid = unsafe { libc::geteuid() }; + let peer_uid = get_peer_uid_unix(stream).await?; + if peer_uid == our_euid || peer_uid == 0 { + debug!( + "Peer uid check passed: peer_uid={}, our_euid={}", + peer_uid, our_euid + ); + Ok(()) + } else { + Err(anyhow!( + "Expect peer uid {} (or root), got {}", + our_euid, + peer_uid + )) + } +} + +#[repr(C)] +struct Ucred { + #[cfg(target_os = "macos")] + ucred: libc::xucred, + + #[cfg(not(target_os = "macos"))] + ucred: libc::ucred, +} +impl Ucred { + #[cfg(target_os = "macos")] + const LEVEL: libc::c_int = libc::SOL_LOCAL; + #[cfg(target_os = "macos")] + const PEERCRED: libc::c_int = libc::LOCAL_PEERCRED; + + #[cfg(not(target_os = "macos"))] + const LEVEL: libc::c_int = libc::SOL_SOCKET; // protocol independent + #[cfg(not(target_os = "macos"))] + const PEERCRED: libc::c_int = libc::SO_PEERCRED; + + fn uid(&self) -> u32 { + #[cfg(target_os = "macos")] + return self.ucred.cr_uid; + + #[cfg(not(target_os = "macos"))] + return self.ucred.uid; + } +} + +async fn get_peer_uid_unix(stream: &UnixStream) -> anyhow::Result { + let fd = stream.as_fd(); + + let mut cred: MaybeUninit = MaybeUninit::uninit(); + + let mut cred_len = std::mem::size_of_val(&cred) as u32; + + let res = unsafe { + libc::getsockopt( + fd.as_raw_fd(), + Ucred::LEVEL, + Ucred::PEERCRED, // SO_PEERCRED + &mut cred as *mut _ as *mut libc::c_void, + &mut cred_len, + ) + }; + + if res == -1 { + return Err(io::Error::last_os_error()) + .with_context(|| "Call to getsockopt for PEERCRED failed"); + } + + let cred = unsafe { cred.assume_init() }; + let required_len = std::mem::size_of_val(&cred); + if (cred_len as usize) < required_len { + anyhow::bail!( + "Result of getsockopt/PEERCRED: output too small ({} < {})", + cred_len, + required_len + ); + } + + Ok(cred.uid()) +} + +impl SpanMetricsSubmitter for ClientInitResp { + fn submit_metric(&mut self, key: SpanMetricName, value: f64) { + self.metrics.insert(key.0.into(), value); + } + fn submit_meta(&mut self, key: SpanMetaName, value: String) { + self.meta.insert(key.0.into(), value); + } + fn submit_meta_dyn_key(&mut self, key: String, value: String) { + self.meta.insert(key, value); + } + fn submit_metric_dyn_key(&mut self, key: String, value: f64) { + self.metrics.insert(key, value); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rmp_serde::Serializer; + use std::path::PathBuf; + use tokio::io::AsyncWriteExt; + use tokio::net::UnixStream; + + fn serialize_message(command: &T) -> Vec { + let mut buf = Vec::new(); + let mut serializer = Serializer::new(&mut buf); + command.serialize(&mut serializer).unwrap(); + + let size = buf.len() as u32; + let mut full_message = Vec::new(); + full_message.extend_from_slice(b"dds\0"); + full_message.extend_from_slice(&size.to_le_bytes()); + full_message.extend_from_slice(&buf); + full_message + } + + fn make_client_init_message() -> Vec { + let client_init_args = ( + 12345u32, + "1.0.0", + "8.0", + Some(true), + ( + Some("/path/to/rules"), + 1000u64, + 10u32, + Option::<&str>::None, + Some(".*"), + (true, 0.5f64), + ), + (true, PathBuf::from("/dev/shm/remote")), + ("my-service", "production"), + ("session-123", "runtime-456"), + ); + serialize_message(&("client_init", client_init_args)) + } + + /// Tests that demonstrate how Framed behaves + /// with various shutdown scenarios. + mod framed_behavior { + use super::*; + + /// When the client cleanly closes the connection (after writing all data), + /// framed.next() returns None. + #[tokio::test] + async fn test_clean_shutdown_returns_none() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + client.shutdown().await.unwrap(); + + let result = framed.next().await; + assert!( + result.is_none(), + "Clean shutdown should return None, got {:?}", + result + ); + } + + /// When the client sends a complete message then closes cleanly, + /// we get the message first, then None on the next read. + #[tokio::test] + async fn test_message_then_clean_shutdown() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + let msg = make_client_init_message(); + client.write_all(&msg).await.unwrap(); + client.shutdown().await.unwrap(); + + let first = framed.next().await; + assert!( + matches!(first, Some(Ok(protocol::Command::ClientInit(_)))), + "Should receive client_init message, got {:?}", + first + ); + + let second = framed.next().await; + assert!( + second.is_none(), + "After message, clean shutdown should return None, got {:?}", + second + ); + } + + /// When the client drops without shutdown (simulating crash/kill), + /// the behavior depends on whether there's partial data in the buffer. + /// With no partial data, it returns None (same as clean shutdown). + #[tokio::test] + async fn test_drop_without_shutdown_no_partial_data() { + let (server, client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + drop(client); + + let result = framed.next().await; + assert!( + result.is_none(), + "Drop without partial data returns None (indistinguishable from clean shutdown), got {:?}", + result + ); + } + + /// When the client sends partial data (incomplete header) then drops, + /// Framed returns an io::Error with ErrorKind::Other and message + /// "bytes remaining on stream". This is tokio_util's way of indicating + /// the stream closed with incomplete data in the buffer. + #[tokio::test] + async fn test_incomplete_header_then_drop() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + client.write_all(b"dds").await.unwrap(); + drop(client); + + let result = framed.next().await; + match result { + Some(Err(e)) => { + assert_eq!(e.kind(), io::ErrorKind::Other); + let msg = e.to_string(); + assert!( + msg.contains("bytes remaining on stream"), + "Expected 'bytes remaining on stream' error, got: {}", + msg + ); + } + other => panic!("Expected Some(Err(...)), got {:?}", other), + } + } + + /// When the client sends a valid header but incomplete body then drops, + /// Framed returns "bytes remaining on stream" error. + #[tokio::test] + async fn test_incomplete_body_then_drop() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + let mut partial = Vec::new(); + partial.extend_from_slice(b"dds\0"); + partial.extend_from_slice(&100u32.to_le_bytes()); + partial.extend_from_slice(b"partial body"); + + client.write_all(&partial).await.unwrap(); + drop(client); + + let result = framed.next().await; + match result { + Some(Err(e)) => { + assert_eq!(e.kind(), io::ErrorKind::Other); + let msg = e.to_string(); + assert!( + msg.contains("bytes remaining on stream"), + "Expected 'bytes remaining on stream' error, got: {}", + msg + ); + } + other => panic!("Expected Some(Err(...)), got {:?}", other), + } + } + + /// When the client sends an invalid header marker, we get InvalidData error. + #[tokio::test] + async fn test_invalid_header_marker() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + let mut invalid = Vec::new(); + invalid.extend_from_slice(b"bad\0"); + invalid.extend_from_slice(&10u32.to_le_bytes()); + invalid.extend_from_slice(b"0123456789"); + + client.write_all(&invalid).await.unwrap(); + client.shutdown().await.unwrap(); + + let result = framed.next().await; + match result { + Some(Err(e)) => { + assert_eq!( + e.kind(), + io::ErrorKind::InvalidData, + "Invalid marker should give InvalidData, got {:?}", + e.kind() + ); + } + other => panic!("Expected Some(Err(InvalidData)), got {:?}", other), + } + } + + /// When the client sends a valid header but invalid msgpack body, + /// we get InvalidData error from deserialization. + #[tokio::test] + async fn test_invalid_msgpack_body() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + let mut invalid = Vec::new(); + invalid.extend_from_slice(b"dds\0"); + invalid.extend_from_slice(&10u32.to_le_bytes()); + invalid.extend_from_slice(b"not msgpack"); + + client.write_all(&invalid).await.unwrap(); + client.shutdown().await.unwrap(); + + let result = framed.next().await; + match result { + Some(Err(e)) => { + assert_eq!( + e.kind(), + io::ErrorKind::InvalidData, + "Invalid msgpack should give InvalidData, got {:?}", + e.kind() + ); + } + other => panic!("Expected Some(Err(InvalidData)), got {:?}", other), + } + } + + /// When the client sends an unknown command name, we get InvalidData error. + #[tokio::test] + async fn test_unknown_command_name() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + + let msg = serialize_message(&("unknown_command", ())); + client.write_all(&msg).await.unwrap(); + client.shutdown().await.unwrap(); + + let result = framed.next().await; + match result { + Some(Err(e)) => { + assert_eq!( + e.kind(), + io::ErrorKind::InvalidData, + "Unknown command should give InvalidData, got {:?}", + e.kind() + ); + } + other => panic!("Expected Some(Err(InvalidData)), got {:?}", other), + } + } + } + + /// Tests for recv_command error classification + mod recv_command_errors { + use super::*; + + /// Clean shutdown should return CleanShutdown error + #[tokio::test] + async fn test_clean_shutdown_returns_clean_shutdown_error() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + let cancel_token = CancellationToken::new(); + + client.shutdown().await.unwrap(); + + let result = recv_command(&mut framed, &cancel_token).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.is::(), + "Clean shutdown should return CleanShutdown, got: {:?}", + err + ); + } + + /// Incomplete data should return ForcefulDisconnect error + #[tokio::test] + async fn test_incomplete_data_returns_forceful_disconnect() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + let cancel_token = CancellationToken::new(); + + client.write_all(b"dds").await.unwrap(); + drop(client); + + let result = recv_command(&mut framed, &cancel_token).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.is::(), + "Incomplete data should return ForcefulDisconnect, got: {:?}", + err + ); + } + + /// Invalid protocol (bad header) should NOT return ForcefulDisconnect + #[tokio::test] + async fn test_invalid_header_returns_io_error() { + let (server, mut client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + let cancel_token = CancellationToken::new(); + + let mut invalid = Vec::new(); + invalid.extend_from_slice(b"bad\0"); + invalid.extend_from_slice(&10u32.to_le_bytes()); + invalid.extend_from_slice(b"0123456789"); + client.write_all(&invalid).await.unwrap(); + client.shutdown().await.unwrap(); + + let result = recv_command(&mut framed, &cancel_token).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.downcast_ref::().is_some(), + "Should contain io::Error in chain, got: {:?}", + err + ); + } + + /// Cancellation should return CleanShutdown (treated same as clean close) + #[tokio::test] + async fn test_cancellation_returns_clean_shutdown() { + let (server, _client) = UnixStream::pair().unwrap(); + let mut framed = tokio_util::codec::Framed::new(server, protocol::CommandCodec); + let cancel_token = CancellationToken::new(); + + cancel_token.cancel(); + + let result = recv_command(&mut framed, &cancel_token).await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + err.is::(), + "Cancellation should return CleanShutdown, got: {:?}", + err + ); + } + } +} diff --git a/appsec/helper-rust/src/client/attributes.rs b/appsec/helper-rust/src/client/attributes.rs new file mode 100644 index 00000000000..126d0cff65d --- /dev/null +++ b/appsec/helper-rust/src/client/attributes.rs @@ -0,0 +1,297 @@ +use std::collections::HashMap; +use std::io::Write; + +use anyhow::Context; +use base64::Engine; +use flate2::write::GzEncoder; +use flate2::Compression; +use libddwaf::object::{Keyed, WafObject, WafObjectType}; + +use crate::{client::log::warning, telemetry}; + +#[derive(Debug)] +pub struct CollectedWafAttributes { + max_plain_schema_allowed: usize, + max_schema_size: usize, + meta: HashMap, + metrics: HashMap, +} + +impl CollectedWafAttributes { + pub fn new(max_plain_schema_allowed: usize, max_schema_size: usize) -> Self { + Self { + max_plain_schema_allowed, + max_schema_size, + meta: HashMap::new(), + metrics: HashMap::new(), + } + } + pub fn add_attribute(&mut self, attr_kv: &Keyed) { + if let Err(e) = self.try_add_attribute(attr_kv) { + warning!("Failed to add attribute: {}", e); + } + } + + fn try_add_attribute(&mut self, attr_kv: &Keyed) -> anyhow::Result<()> { + let key = attr_kv + .key_str() + .map_err(|e| anyhow::anyhow!("Attribute key is not valid UTF-8: {}", e))?; + let value = attr_kv.value(); + + if key.starts_with("_dd.appsec.s.") { + self.add_schema_attribute(key, value) + } else { + self.add_regular_attribute(key, value) + } + } + + fn add_schema_attribute(&mut self, key: &str, value: &WafObject) -> anyhow::Result<()> { + let mut json_derivative = + serde_json::to_string(value).with_context(|| "Failed to serialize schema value")?; + + if json_derivative.len() > self.max_plain_schema_allowed { + let compressed = compress(&json_derivative)?; + json_derivative = base64::engine::general_purpose::STANDARD.encode(&compressed); + } + + if json_derivative.len() > self.max_schema_size { + anyhow::bail!( + "Schema for key {} is too large ({} bytes > {} max)", + key, + json_derivative.len(), + self.max_schema_size + ); + } + + self.meta.insert(key.to_owned(), json_derivative); + Ok(()) + } + + fn add_regular_attribute(&mut self, key: &str, value: &WafObject) -> anyhow::Result<()> { + match value.object_type() { + WafObjectType::Signed => { + let val = value + .as_type::() + .unwrap() + .value() as f64; + self.metrics.insert(key.to_owned(), val); + } + WafObjectType::Unsigned => { + let val = value + .as_type::() + .unwrap() + .value() as f64; + self.metrics.insert(key.to_owned(), val); + } + WafObjectType::Float => { + let val = value + .as_type::() + .unwrap() + .value(); + self.metrics.insert(key.to_owned(), val); + } + WafObjectType::String => { + let val = value.as_type::().unwrap(); + let s = val + .as_str() + .with_context(|| "String value is not valid UTF-8")?; + self.meta.insert(key.to_owned(), s.to_owned()); + } + other => { + anyhow::bail!("Unsupported attribute type: {:?}", other); + } + } + Ok(()) + } +} + +impl telemetry::SpanMetricsGenerator for CollectedWafAttributes { + fn generate_span_metrics(&'_ self, submitter: &mut dyn telemetry::SpanMetricsSubmitter) { + for (key, value) in &self.meta { + submitter.submit_meta_dyn_key(key.clone(), value.clone()); + } + + for (key, value) in &self.metrics { + submitter.submit_metric_dyn_key(key.clone(), *value); + } + } +} + +fn compress(data: &str) -> anyhow::Result> { + if data.is_empty() { + anyhow::bail!("Cannot compress empty data"); + } + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(data.as_bytes()) + .with_context(|| "Failed to write data to gzip encoder")?; + encoder + .finish() + .with_context(|| "Failed to finish gzip compression") +} + +#[cfg(test)] +mod tests { + use super::*; + use base64::Engine; + use flate2::read::GzDecoder; + use libddwaf::waf_map; + use std::io::Read; + + #[test] + fn test_add_numeric_signed_attribute() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let map = waf_map! {("test.metric", -42i64)}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.metrics.len(), 1); + assert_eq!(attrs.metrics.get("test.metric"), Some(&-42.0)); + assert_eq!(attrs.meta.len(), 0); + } + + #[test] + fn test_add_numeric_unsigned_attribute() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let map = waf_map! {("test.metric", 100u64)}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.metrics.len(), 1); + assert_eq!(attrs.metrics.get("test.metric"), Some(&100.0)); + assert_eq!(attrs.meta.len(), 0); + } + + #[test] + fn test_add_numeric_float_attribute() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let map = waf_map! {("test.metric", 3.14)}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.metrics.len(), 1); + assert_eq!(attrs.metrics.get("test.metric"), Some(&3.14)); + assert_eq!(attrs.meta.len(), 0); + } + + #[test] + fn test_add_string_attribute() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let map = waf_map! {("test.meta", "value")}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.meta.len(), 1); + assert_eq!(attrs.meta.get("test.meta"), Some(&"value".to_string())); + assert_eq!(attrs.metrics.len(), 0); + } + + #[test] + fn test_add_small_schema_attribute() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let map = waf_map! {("_dd.appsec.s.req.body", "small")}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.meta.len(), 1); + let stored = attrs.meta.get("_dd.appsec.s.req.body").unwrap(); + assert_eq!(stored, "\"small\""); + } + + #[test] + fn test_add_large_schema_attribute_gets_compressed() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + let large_string = "x".repeat(300); + let large_str = large_string.as_str(); + let map = waf_map! {("_dd.appsec.s.res.body", large_str)}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.meta.len(), 1); + let stored = attrs.meta.get("_dd.appsec.s.res.body").unwrap(); + + let decoded = base64::engine::general_purpose::STANDARD + .decode(stored) + .expect("Should be base64 encoded"); + let mut decoder = GzDecoder::new(&decoded[..]); + let mut decompressed = String::new(); + decoder + .read_to_string(&mut decompressed) + .expect("Should be gzip compressed"); + + let expected_json = format!("\"{}\"", large_string); + assert_eq!(decompressed, expected_json); + } + + #[test] + fn test_schema_attribute_too_large_gets_rejected() { + let mut attrs = CollectedWafAttributes::new(260, 10); + let large_string = "x".repeat(300); + let large_str = large_string.as_str(); + let map = waf_map! {("_dd.appsec.s.huge", large_str)}; + let attr = map.into_iter().next().unwrap(); + + attrs.add_attribute(&attr); + + assert_eq!(attrs.meta.len(), 0); + } + + #[test] + fn test_generate_metrics() { + let mut attrs = CollectedWafAttributes::new(260, 25000); + + let map = waf_map! {("metric1", 10i64)}; + attrs.add_attribute(&map.into_iter().next().unwrap()); + + let map = waf_map! {("metric2", 20.5)}; + attrs.add_attribute(&map.into_iter().next().unwrap()); + + let map = waf_map! {("meta1", "value1")}; + attrs.add_attribute(&map.into_iter().next().unwrap()); + + let map = waf_map! {("_dd.appsec.s.test", "schema")}; + attrs.add_attribute(&map.into_iter().next().unwrap()); + + use crate::telemetry::{SpanMetricsGenerator, SpanMetricsSubmitter}; + + struct TestSubmitter { + meta: HashMap, + metrics: HashMap, + } + impl SpanMetricsSubmitter for TestSubmitter { + fn submit_metric(&mut self, _key: crate::telemetry::SpanMetricName, _value: f64) {} + fn submit_meta(&mut self, _key: crate::telemetry::SpanMetaName, _value: String) {} + fn submit_meta_dyn_key(&mut self, key: String, value: String) { + self.meta.insert(key, value); + } + fn submit_metric_dyn_key(&mut self, key: String, value: f64) { + self.metrics.insert(key, value); + } + } + + let mut submitter = TestSubmitter { + meta: HashMap::new(), + metrics: HashMap::new(), + }; + + attrs.generate_span_metrics(&mut submitter); + + assert_eq!(submitter.metrics.len(), 2); + assert_eq!(submitter.metrics.get("metric1"), Some(&10.0)); + assert_eq!(submitter.metrics.get("metric2"), Some(&20.5)); + + assert_eq!(submitter.meta.len(), 2); + assert_eq!(submitter.meta.get("meta1"), Some(&"value1".to_string())); + assert_eq!( + submitter.meta.get("_dd.appsec.s.test"), + Some(&"\"schema\"".to_string()) + ); + } +} diff --git a/appsec/helper-rust/src/client/log.rs b/appsec/helper-rust/src/client/log.rs new file mode 100644 index 00000000000..af7c58bcd29 --- /dev/null +++ b/appsec/helper-rust/src/client/log.rs @@ -0,0 +1,208 @@ +use std::backtrace::Backtrace; + +use futures::Future; +use tokio::task_local; + +task_local! { + pub static CLIENT_ID: u64; +} + +/// Key used to pass anyhow backtrace through log's key-value API +pub const ANYHOW_BACKTRACE_KEY: &str = "__anyhow_backtrace"; + +/// Logs an error with an optional backtrace, forcing the emitted record location. +#[doc(hidden)] +pub fn log_error_with_backtrace_at( + file: &'static str, + line: u32, + module_path: &'static str, + msg: &str, + backtrace: Option<&Backtrace>, +) { + if !log::log_enabled!(log::Level::Error) { + return; + } + + let formatted_msg = if let Ok(client_id) = CLIENT_ID.try_with(|id| *id) { + format!("Client #{}: {}", client_id, msg) + } else { + msg.to_string() + }; + + struct BacktraceKvs<'a> { + bt: &'a Backtrace, + } + + impl<'kvs> log::kv::Source for BacktraceKvs<'kvs> { + fn visit<'a>( + &'a self, + visitor: &mut dyn log::kv::VisitSource<'a>, + ) -> Result<(), log::kv::Error> { + visitor.visit_pair( + log::kv::Key::from_str(ANYHOW_BACKTRACE_KEY), + log::kv::Value::from_display(self.bt), + ) + } + } + + struct EmptSource; + impl log::kv::Source for EmptSource { + fn visit<'a>( + &'a self, + _visitor: &mut dyn log::kv::VisitSource<'a>, + ) -> Result<(), log::kv::Error> { + Ok(()) + } + } + + let kvs: Box = if let Some(bt) = backtrace { + Box::new(BacktraceKvs { bt }) + } else { + Box::new(EmptSource) + }; + + log::logger().log( + &log::Record::builder() + .args(format_args!("{}", formatted_msg)) + .level(log::Level::Error) + .target(module_path) + .module_path(Some(module_path)) + .file(Some(file)) + .line(Some(line)) + .key_values(&kvs) + .build(), + ); +} + +pub trait TryGetBacktrace { + fn try_get_backtrace(&self) -> Option<&Backtrace>; +} + +impl TryGetBacktrace for anyhow::Error { + #[inline] + fn try_get_backtrace(&self) -> Option<&Backtrace> { + Some(self.backtrace()) + } +} + +impl TryGetBacktrace for &T { + #[inline] + fn try_get_backtrace(&self) -> Option<&Backtrace> { + None + } +} + +macro_rules! client_log { + ($level:ident, $($arg:tt)*) => { + if let Ok(client_id) = $crate::client::log::CLIENT_ID.try_with(|id| *id) { + ::log::$level!("Client #{}: {}", client_id, format!($($arg)*)); + } else { + ::log::$level!("{}", format!($($arg)*)); + } + }; +} +pub(crate) use client_log; + +macro_rules! client_log_gen { + ($level:expr, $($arg:tt)*) => { + match $level { + ::log::Level::Trace => crate::client::log::trace!($($arg)*), + ::log::Level::Debug => crate::client::log::debug!($($arg)*), + ::log::Level::Info => crate::client::log::info!($($arg)*), + ::log::Level::Warn => crate::client::log::warning!($($arg)*), + ::log::Level::Error => crate::client::log::error!($($arg)*), + } + }; +} +pub(crate) use client_log_gen; + +macro_rules! trace { + ($($arg:tt)*) => { crate::client::log::client_log!(trace, $($arg)*) }; +} +pub(crate) use trace; + +macro_rules! debug { + ($($arg:tt)*) => { crate::client::log::client_log!(debug, $($arg)*) }; +} +pub(crate) use debug; + +macro_rules! info { + ($($arg:tt)*) => { crate::client::log::client_log!(info, $($arg)*) }; +} +pub(crate) use info; + +macro_rules! warning { + ($($arg:tt)*) => { crate::client::log::client_log!(warn, $($arg)*) }; +} +pub(crate) use warning; + +#[macro_export] +macro_rules! error { + // No arguments - just format string + ($fmt:literal) => {{ + $crate::client::log::log_error_with_backtrace_at( + file!(), + line!(), + module_path!(), + $fmt, + None, + ); + }}; + + // With arguments - check for anyhow backtrace + ($fmt:literal, $($arg:expr),* $(,)?) => {{ + use $crate::client::log::TryGetBacktrace; + let __msg = format!($fmt, $($arg),*); + + let mut __found = false; + $( + if !__found { + if let Some(bt) = (&$arg).try_get_backtrace() { + $crate::client::log::log_error_with_backtrace_at( + file!(), + line!(), + module_path!(), + &__msg, + Some(bt), + ); + __found = true; + } + } + )* + if !__found { + $crate::client::log::log_error_with_backtrace_at( + file!(), + line!(), + module_path!(), + &__msg, + None, + ); + } + }}; +} +pub(crate) use error; + +pub async fn with_scoped_client_id(client_id: u64, fut: F) +where + F: Future, +{ + CLIENT_ID.scope(client_id, fut).await; +} + +pub fn fmt_bin(vec: &[u8]) -> impl std::fmt::Debug + '_ { + struct BinFormatter<'a>(&'a [u8]); + impl<'a> std::fmt::Debug for BinFormatter<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for &c in self.0 { + if c.is_ascii_graphic() { + write!(f, "{}", c as char)?; + } else { + write!(f, "\\x{:02x}", c)?; + } + } + Ok(()) + } + } + + BinFormatter(vec) +} diff --git a/appsec/helper-rust/src/client/metrics.rs b/appsec/helper-rust/src/client/metrics.rs new file mode 100644 index 00000000000..b4d433ddb83 --- /dev/null +++ b/appsec/helper-rust/src/client/metrics.rs @@ -0,0 +1,232 @@ +use std::{borrow::Cow, collections::HashMap, time::Duration}; + +use crate::telemetry; + +#[derive(Default, Debug)] +pub struct CollectingMetricsSubmitter { + meta: HashMap, String>, + metrics: HashMap, f64>, +} +impl CollectingMetricsSubmitter { + pub fn take_metrics(&mut self) -> HashMap, f64> { + std::mem::take(&mut self.metrics) + } + pub fn take_meta(&mut self) -> HashMap, String> { + std::mem::take(&mut self.meta) + } +} +impl telemetry::SpanMetricsSubmitter for CollectingMetricsSubmitter { + fn submit_metric(&mut self, key: telemetry::SpanMetricName, value: f64) { + self.metrics.insert(key.0.into(), value); + } + fn submit_meta(&mut self, key: telemetry::SpanMetaName, value: String) { + self.meta.insert(key.0.into(), value); + } + fn submit_meta_dyn_key(&mut self, key: String, value: String) { + self.meta.insert(key.into(), value); + } + fn submit_metric_dyn_key(&mut self, key: String, value: f64) { + self.metrics.insert(key.into(), value); + } +} + +#[derive(Default, Debug)] +pub struct WafMetrics { + // Ruleset version (context for tag generation) + rules_version: Option, + + /// Whether a non-RASP evaluation hit an error + waf_hit_error: bool, + + /// Total WAF execution time in milliseconds (non-RASP calls only) + waf_duration: Duration, + + /// Whether the WAF hit a timeout during non-RASP calls + waf_hit_timeout: bool, + + /// Total RASP execution time in milliseconds + rasp_duration: Duration, + + /// Count of RASP rule evaluations + rasp_rule_evals: u32, + + /// Count of RASP timeouts + rasp_timeouts: u32, + + /// Per-rule-type RASP metrics for telemetry + rasp_per_rule: HashMap, + + /// Whether the WAF triggered any rules + had_triggers: bool, + + /// Whether the request was blocked + request_blocked: bool, + + /// Whether the input was truncated by the extension + input_truncated: bool, +} + +#[derive(Default, Debug, Clone)] +pub struct RaspRuleMetrics { + pub evals: u32, + pub matches: u32, + pub timeouts: u32, +} + +impl WafMetrics { + pub fn new(rules_version: Option) -> Self { + Self { + rules_version, + waf_hit_error: false, + waf_duration: Duration::ZERO, + waf_hit_timeout: false, + rasp_duration: Duration::ZERO, + rasp_rule_evals: 0, + rasp_timeouts: 0, + rasp_per_rule: HashMap::new(), + had_triggers: false, + request_blocked: false, + input_truncated: false, + } + } + + pub fn set_input_truncated(&mut self, input_truncated: bool) { + self.input_truncated = input_truncated; + } + + pub fn record_non_rasp_error_eval(&mut self) { + self.waf_hit_error = true; + } + + pub fn record_non_rasp_eval(&mut self, run_output: &libddwaf::RunOutput) { + self.waf_duration += run_output.duration(); + + if run_output.timeout() { + self.waf_hit_timeout = true; + } + if run_output.has_events() { + self.had_triggers = true; + } + if run_output.is_blocking() { + self.request_blocked = true; + } + } + + pub fn record_rasp_eval(&mut self, rule_type: &str, run_output: &libddwaf::RunOutput) { + self.rasp_duration += run_output.duration(); + self.rasp_rule_evals += 1; + + if run_output.timeout() { + self.rasp_timeouts += 1; + } + + let entry = self.rasp_per_rule.entry(rule_type.to_string()).or_default(); + entry.evals += 1; + + if run_output.has_events() { + entry.matches += 1; + } + if run_output.is_blocking() { + self.request_blocked = true; + } + } +} +trait RunOutputExt { + fn has_events(&self) -> bool; + fn is_blocking(&self) -> bool; +} +impl RunOutputExt for libddwaf::RunOutput { + fn has_events(&self) -> bool { + self.events() + .is_some_and(|events| !events.value().is_empty()) + } + fn is_blocking(&self) -> bool { + self.actions().is_some_and(|actions| { + actions + .value() + .iter() + .any(|action| matches!(action.key().to_str(), Some("block") | Some("redirect"))) + }) + } +} +impl telemetry::TelemetryMetricsGenerator for WafMetrics { + fn generate_telemetry_metrics( + &'_ self, + submitter: &mut dyn telemetry::TelemetryMetricSubmitter, + ) { + // waf.requests metrics + let mut tags = telemetry::TelemetryTags::new(); + tags.add("waf_version", crate::service::Service::waf_version()); + if let Some(ref rules_ver) = self.rules_version { + tags.add("event_rules_version", rules_ver); + } + if self.had_triggers { + tags.add("rule_triggered", "true"); + } + if self.request_blocked { + tags.add("request_blocked", "true"); + } + if self.waf_hit_timeout { + tags.add("waf_timeout", "true"); + } + if self.input_truncated { + tags.add("input_truncated", "true"); + } + submitter.submit_metric(telemetry::WAF_REQUESTS, 1.0, tags); + + // Rasp rule metrics + for (rule_type, metrics) in &self.rasp_per_rule { + let mut tags = telemetry::TelemetryTags::new(); + tags.add("rule_type", rule_type); + tags.add("waf_version", crate::service::Service::waf_version()); + + if metrics.evals > 0 { + submitter.submit_metric( + telemetry::RASP_RULE_EVAL, + metrics.evals as f64, + tags.clone(), + ); + } + + if metrics.matches > 0 { + submitter.submit_metric( + telemetry::RASP_RULE_MATCH, + metrics.matches as f64, + tags.clone(), + ); + } + + // tests expect this to always be sent, even if 0 + submitter.submit_metric(telemetry::RASP_TIMEOUT, metrics.timeouts as f64, tags); + } + } +} + +impl telemetry::SpanMetricsGenerator for WafMetrics { + fn generate_span_metrics(&'_ self, submitter: &mut dyn telemetry::SpanMetricsSubmitter) { + if !self.waf_duration.is_zero() { + submitter.submit_metric(telemetry::WAF_DURATION, self.waf_duration.duration_ms_f64()); + } + if !self.rasp_duration.is_zero() { + submitter.submit_metric( + telemetry::RAST_DURATION, + self.rasp_duration.duration_ms_f64(), + ); + } + if self.rasp_rule_evals > 0 { + submitter.submit_metric(telemetry::RAST_RULE_EVALS, self.rasp_rule_evals as f64); + } + if self.rasp_timeouts > 0 { + submitter.submit_metric(telemetry::RAST_TIMEOUTS, self.rasp_timeouts as f64); + } + } +} + +trait DurationExt { + fn duration_ms_f64(&self) -> f64; +} +impl DurationExt for Duration { + fn duration_ms_f64(&self) -> f64 { + self.as_secs() as f64 * 1_000.0 + self.subsec_nanos() as f64 / 1_000_000.0 + } +} diff --git a/appsec/helper-rust/src/client/protocol.rs b/appsec/helper-rust/src/client/protocol.rs new file mode 100644 index 00000000000..59a606d516b --- /dev/null +++ b/appsec/helper-rust/src/client/protocol.rs @@ -0,0 +1,720 @@ +use rmp_serde::Deserializer; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Serialize}; +use serde_tuple::{Deserialize_tuple, Serialize_tuple}; +use std::borrow::Cow; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::io; +use std::path::{Path, PathBuf}; +use tokio_util::bytes::{Buf, BytesMut}; +use tokio_util::codec::{Decoder, Encoder}; + +use crate::client::log::{fmt_bin, trace}; + +pub const VERSION_FOR_PROTO: &str = "1.16.0"; +const MAX_MESSAGE_SIZE: u32 = 4 * 1024 * 1024; + +#[derive(Debug)] +pub enum Command { + ClientInit(Box), + ConfigSync(Box), + RequestInit(Box), + RequestExec(Box), + RequestShutdown(Box), +} + +#[derive(Debug)] +pub enum CommandResponse<'a> { + ProtocolError, + ClientInit(ClientInitResp), + ConfigSync, + ConfigFeatures(ConfigFeaturesResp), + RequestInit(RequestInitResp<'a>), + RequestExec(RequestExecResp), + RequestShutdown(RequestShutdownResp), +} + +#[derive(Debug, Deserialize_tuple)] +pub struct ClientInitArgs { + #[allow(dead_code)] + pub pid: u32, + #[allow(dead_code)] + pub ddappsec_version: String, + #[allow(dead_code)] + pub php_version: String, + pub appsec_enabled: Option, + pub waf_config: WafSettings, + pub remote_config: RemoteConfigSettings, + pub telemetry_settings: TelemetrySettings, + pub sidecar_settings: SidecarSettings, +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug, Deserialize)] +pub struct TelemetrySettings { + pub service_name: String, + pub env_name: String, +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug, Deserialize)] +pub struct SidecarSettings { + pub session_id: String, + pub runtime_id: String, +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug, Deserialize)] +pub struct WafSettings { + #[serde(deserialize_with = "empty_string_as_none")] + pub rules_file: Option, + #[serde(deserialize_with = "zero_as_none")] + pub waf_timeout_us: Option, + pub trace_rate_limit: u32, + #[serde(deserialize_with = "empty_string_as_none")] + pub obfuscator_key_regex: Option, + #[serde(deserialize_with = "empty_string_as_none")] + pub obfuscator_value_regex: Option, + pub schema_extraction: SchemaExtraction, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SchemaExtraction { + pub enabled: bool, + pub sampling_period: f64, +} +impl PartialEq for SchemaExtraction { + fn eq(&self, other: &Self) -> bool { + self.enabled == other.enabled + && self.sampling_period.to_bits() == other.sampling_period.to_bits() + } +} +impl Eq for SchemaExtraction {} +impl Hash for SchemaExtraction { + fn hash(&self, state: &mut H) { + self.enabled.hash(state); + self.sampling_period.to_bits().hash(state); + } +} + +#[derive(PartialEq, Eq, Hash, Clone, Debug, Deserialize)] +pub struct RemoteConfigSettings { + enabled: bool, + shmem_path: PathBuf, +} + +impl RemoteConfigSettings { + pub fn new(enabled: bool, shmem_path: PathBuf) -> Self { + Self { + enabled, + shmem_path, + } + } + + fn enabled(&self) -> bool { + self.enabled && !self.shmem_path.as_os_str().is_empty() + } + + pub fn shmem_path(&self) -> Option<&Path> { + if self.enabled() { + Some(&self.shmem_path) + } else { + None + } + } +} + +#[derive(Debug, Default, Serialize_tuple)] +pub struct ClientInitResp { + pub status: String, + pub version: &'static str, + pub errors: Vec, + pub meta: HashMap, + pub metrics: HashMap, + pub helper_runtime: Option, +} + +#[derive(Debug, Deserialize_tuple)] +pub struct ConfigSyncArgs { + pub rem_cfg_path: String, + pub telemetry_settings: TelemetrySettings, +} + +#[derive(Debug, Serialize_tuple)] +pub struct ConfigFeaturesResp { + pub enabled: bool, +} + +#[derive(Debug, Deserialize_tuple)] +pub struct RequestInitArgs { + pub data: libddwaf::object::WafMap, +} + +#[derive(Debug, Serialize_tuple)] +pub struct RequestInitResp<'a> { + pub actions: &'a Vec, + pub triggers: &'a Vec, + pub force_keep: bool, + pub settings: HashMap<&'static str, String>, +} +#[derive(Debug, Serialize_tuple)] +pub struct ActionInstance { + pub action: &'static str, + pub parameters: HashMap, +} + +#[derive(Debug, Deserialize_tuple)] +pub struct RequestExecArgs { + pub data: libddwaf::object::WafMap, + pub options: RequestExecOptions, +} +#[derive(Debug, Deserialize)] +pub struct RequestExecOptions { + #[serde(rename = "rasp_rule")] + pub run_type: WafRunType, + pub subctx_id: Option, + #[serde(default)] + pub subctx_last_call: bool, +} +impl RequestExecOptions { + pub fn regular() -> Self { + Self { + run_type: WafRunType::NonRasp, + subctx_id: None, + subctx_last_call: false, + } + } +} +#[derive(Debug, PartialEq)] +pub enum WafRunType { + NonRasp, + RaspRule(String), +} +impl<'de> Deserialize<'de> for WafRunType { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let opt = Option::::deserialize(deserializer)?; + + match opt.as_deref() { + None | Some("") => Ok(WafRunType::NonRasp), + Some(s) => Ok(WafRunType::RaspRule(s.to_string())), + } + } +} + +#[derive(Debug, Serialize_tuple)] +pub struct RequestExecResp { + pub actions: Vec, + pub triggers: Vec, + pub force_keep: bool, + pub settings: HashMap, +} + +#[derive(Debug, Deserialize_tuple)] +pub struct RequestShutdownArgs { + pub data: libddwaf::object::WafMap, + pub api_sec_samp_key: u64, + #[allow(dead_code)] + pub queue_id: u64, // TODO: unused, update protocol + pub input_truncated: bool, +} + +#[derive(Debug, Serialize_tuple)] +pub struct RequestShutdownResp { + pub actions: Vec, + pub triggers: Vec, + pub force_keep: bool, + pub settings: HashMap, + pub meta: HashMap, String>, + pub metrics: HashMap, f64>, +} + +impl<'de> Deserialize<'de> for Command { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct CommandVisitor; + + impl<'de> serde::de::Visitor<'de> for CommandVisitor { + type Value = Command; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("an array with form [command_name, [command_args...]]") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let command_name: String = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(0, &self))?; + + match command_name.as_str() { + "client_init" => { + let args: ClientInitArgs = seq.next_element()?.ok_or_else(|| { + serde::de::Error::custom("Missing arguments for ClientInit") + })?; + Ok(Command::ClientInit(Box::new(args))) + } + "config_sync" => { + let args: ConfigSyncArgs = seq.next_element()?.ok_or_else(|| { + serde::de::Error::custom("Missing arguments for ConfigSync") + })?; + Ok(Command::ConfigSync(Box::new(args))) + } + "request_init" => { + let args: RequestInitArgs = seq.next_element()?.ok_or_else(|| { + serde::de::Error::custom("Missing arguments for RequestInit") + })?; + Ok(Command::RequestInit(Box::new(args))) + } + "request_exec" => { + let args: RequestExecArgs = seq.next_element()?.ok_or_else(|| { + serde::de::Error::custom("Missing arguments for RequestExec") + })?; + Ok(Command::RequestExec(Box::new(args))) + } + "request_shutdown" => { + let args: RequestShutdownArgs = seq.next_element()?.ok_or_else(|| { + serde::de::Error::custom("Missing arguments for RequestShutdown") + })?; + Ok(Command::RequestShutdown(Box::new(args))) + } + v => Err(serde::de::Error::custom(format!( + "Got unknown command name {}", + v + ))), + } + } + } + + deserializer.deserialize_seq(CommandVisitor) + } +} + +fn empty_string_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let opt: Option = Option::deserialize(deserializer)?; + Ok(opt.filter(|s| !s.is_empty())) +} + +fn zero_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value: u64 = u64::deserialize(deserializer)?; + Ok(if value == 0 { None } else { Some(value) }) +} + +#[repr(C)] +struct Header { + marker: [u8; 4], + size: u32, +} +impl Header { + const VALID_MARKER: [u8; 4] = *b"dds\0"; + fn is_valid_marker(&self) -> bool { + self.marker == Self::VALID_MARKER + } + fn as_slice(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts(self as *const _ as *const u8, std::mem::size_of::()) + } + } +} + +pub struct CommandCodec; +impl Decoder for CommandCodec { + type Item = Command; + + type Error = io::Error; + + fn decode( + &mut self, + src: &mut tokio_util::bytes::BytesMut, + ) -> Result, Self::Error> { + if src.len() < std::mem::size_of::
() { + return Ok(None); + } + + let header_bytes = &src[..std::mem::size_of::
()]; + let header = unsafe { std::ptr::read_unaligned(header_bytes.as_ptr() as *const Header) }; + if !header.is_valid_marker() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Invalid header marker", + )); + } + + if header.size > MAX_MESSAGE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Message is too large: {} bytes (supported up to 4 MB)", + header.size + ), + )); + } + + let total_size = std::mem::size_of::
() + header.size as usize; + + if src.len() < total_size { + if src.capacity() < total_size { + src.reserve(total_size - src.capacity()); + } + return Ok(None); + } + + let data = &src[std::mem::size_of::
()..total_size]; + + trace!( + "Decoding message with size {}: {:?}", + header.size, + fmt_bin(data) + ); + + let mut de = Deserializer::from_read_ref(data); + let cmd = Command::deserialize(&mut de) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + src.advance(total_size); + + Ok(Some(cmd)) + } +} + +impl Serialize for CommandResponse<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut state = serializer.serialize_seq(Some(2))?; + match self { + CommandResponse::ProtocolError => { + state.serialize_element("error")?; + state.serialize_element(&())?; + state.end() + } + CommandResponse::ConfigSync => { + state.serialize_element("config_sync")?; + state.serialize_element(&())?; + state.end() + } + CommandResponse::ConfigFeatures(resp) => { + state.serialize_element("config_features")?; + state.serialize_element(resp)?; + state.end() + } + CommandResponse::ClientInit(resp) => { + state.serialize_element("client_init")?; + state.serialize_element(resp)?; + state.end() + } + CommandResponse::RequestInit(resp) => { + state.serialize_element("request_init")?; + state.serialize_element(resp)?; + state.end() + } + CommandResponse::RequestExec(resp) => { + state.serialize_element("request_exec")?; + state.serialize_element(resp)?; + state.end() + } + CommandResponse::RequestShutdown(resp) => { + state.serialize_element("request_shutdown")?; + state.serialize_element(resp)?; + state.end() + } + } + } +} + +impl Encoder> for CommandCodec { + type Error = io::Error; + + fn encode(&mut self, item: CommandResponse<'_>, dst: &mut BytesMut) -> Result<(), Self::Error> { + let mut buf = Vec::new(); + let mut serializer = rmp_serde::Serializer::new(&mut buf); + + // The protocol supports responding with several messages, but actually + // only one message is ever sent (see command_helpers.c) + [item] + .serialize(&mut serializer) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + let size = buf.len(); + if size > MAX_MESSAGE_SIZE as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "Message is too large: {} bytes (supported up to 4 MB)", + size + ), + )); + } + + let size = size as u32; + let header = Header { + marker: Header::VALID_MARKER, + size, + }; + + trace!("Encoding message with size {}: {:?}", size, fmt_bin(&buf)); + + dst.extend_from_slice(header.as_slice()); + dst.reserve(size as usize); + dst.extend_from_slice(&buf); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use libddwaf::waf_map; + use rmp_serde::Serializer; + use tokio_util::bytes::BytesMut; + + #[tokio::test] + async fn test_command_deserialization() { + fn serialize_message(command: &T) -> Vec { + let mut buf = Vec::new(); + let mut serializer = Serializer::new(&mut buf); + command.serialize(&mut serializer).unwrap(); + + let size = buf.len() as u32; + let mut full_message = Vec::new(); + + full_message.extend_from_slice(b"dds\0"); + full_message.extend_from_slice(&size.to_le_bytes()); + + full_message.extend_from_slice(&buf); + + full_message + } + + let client_init_args = ( + 12345, + "1.0.0", + "8.0", + Some(true), + ( + Some("/path/to/rules"), + 1000, + 10, + Option::<&str>::None, + Some(".*"), + (true, 0.5), + ), + (true, PathBuf::from("/dev/shm/remote")), + ("my-service", "production"), + ("session-123", "runtime-456"), + ); + + let valid_command = ("client_init", client_init_args); + let valid_data = serialize_message(&valid_command); + + let mut decoder = CommandCodec; + let mut buf = BytesMut::new(); + + buf.extend_from_slice(&valid_data); + let decoded = decoder.decode(&mut buf).unwrap(); + println!("{:?}", decoded); + assert!(matches!(decoded, Some(Command::ClientInit(_)))); + } + + #[tokio::test] + async fn test_command_response_serialization() { + let resp = CommandResponse::ClientInit(ClientInitResp { + status: "ok".to_string(), + version: "1.0.0", + errors: vec![], + meta: HashMap::new(), + metrics: HashMap::new(), + helper_runtime: None, + }); + + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + } + + fn serialize_message(command: &T) -> Vec { + let mut buf = Vec::new(); + let mut serializer = Serializer::new(&mut buf); + command.serialize(&mut serializer).unwrap(); + + let size = buf.len() as u32; + let mut full_message = Vec::new(); + + full_message.extend_from_slice(b"dds\0"); + full_message.extend_from_slice(&size.to_le_bytes()); + full_message.extend_from_slice(&buf); + + full_message + } + + #[tokio::test] + async fn test_config_sync_command() { + let config_sync_args = ("/path/to/config", ("service", "env")); + let command = ("config_sync", config_sync_args); + let data = serialize_message(&command); + + let mut decoder = CommandCodec; + let mut buf = BytesMut::new(); + buf.extend_from_slice(&data); + + let decoded = decoder.decode(&mut buf).unwrap(); + assert!(matches!(decoded, Some(Command::ConfigSync(_)))); + if let Some(Command::ConfigSync(args)) = decoded { + assert_eq!(args.rem_cfg_path, "/path/to/config"); + assert_eq!(args.telemetry_settings.service_name, "service"); + assert_eq!(args.telemetry_settings.env_name, "env"); + } + } + + #[tokio::test] + async fn test_config_sync_response() { + let resp = CommandResponse::ConfigSync; + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } + + #[tokio::test] + async fn test_config_features_response() { + let resp = CommandResponse::ConfigFeatures(ConfigFeaturesResp { enabled: true }); + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } + + #[tokio::test] + async fn test_request_init_command() { + let waf_map = waf_map!(("foo", "bar"),); + let command = ("request_init", (&waf_map,)); + let data = serialize_message(&command); + + let mut decoder = CommandCodec; + let mut buf = BytesMut::new(); + buf.extend_from_slice(&data); + + let decoded = decoder.decode(&mut buf).unwrap(); + assert!(matches!(decoded, Some(Command::RequestInit(_)))); + if let Some(Command::RequestInit(args)) = decoded { + assert_eq!(args.data, waf_map); + } + } + + #[tokio::test] + async fn test_request_init_response() { + let actions = vec![ActionInstance { + action: "block", + parameters: HashMap::from([("type".to_string(), "auto".to_string())]), + }]; + let triggers = vec!["trigger1".to_string()]; + let resp = CommandResponse::RequestInit(RequestInitResp { + actions: &actions, + triggers: &triggers, + force_keep: true, + settings: HashMap::from([("setting1", "value1".to_string())]), + }); + + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } + + #[tokio::test] + async fn test_request_exec_command() { + let waf_map = waf_map!(("foo", "bar"),); + let options = (Some("rasp_rule"), Some("subctx_id"), false); + + let command = ("request_exec", (&waf_map, options)); + let data = serialize_message(&command); + + let mut decoder = CommandCodec; + let mut buf = BytesMut::new(); + buf.extend_from_slice(&data); + + let decoded = decoder.decode(&mut buf).unwrap(); + assert!(matches!(decoded, Some(Command::RequestExec(_)))); + if let Some(Command::RequestExec(args)) = decoded { + assert_eq!(args.data, waf_map); + assert_eq!( + args.options.run_type, + WafRunType::RaspRule("rasp_rule".to_string()) + ); + assert_eq!(args.options.subctx_id, Some("subctx_id".to_string())); + assert!(!args.options.subctx_last_call); + } + } + + #[tokio::test] + async fn test_request_exec_response() { + let resp = CommandResponse::RequestExec(RequestExecResp { + actions: vec![], + triggers: vec![], + force_keep: false, + settings: HashMap::new(), + }); + + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } + + #[tokio::test] + async fn test_request_shutdown_command() { + let waf_map = waf_map!(("foo", "bar"),); + let command = ("request_shutdown", (&waf_map, 12345u64, 67890u64, true)); + let data = serialize_message(&command); + + let mut decoder = CommandCodec; + let mut buf = BytesMut::new(); + buf.extend_from_slice(&data); + + let decoded = decoder.decode(&mut buf).unwrap(); + assert!(matches!(decoded, Some(Command::RequestShutdown(_)))); + if let Some(Command::RequestShutdown(args)) = decoded { + assert_eq!(args.data, waf_map); + assert_eq!(args.api_sec_samp_key, 12345); + assert_eq!(args.queue_id, 67890); + assert!(args.input_truncated); + } + } + + #[tokio::test] + async fn test_request_shutdown_response() { + let resp = CommandResponse::RequestShutdown(RequestShutdownResp { + actions: vec![], + triggers: vec![], + force_keep: false, + settings: HashMap::new(), + meta: HashMap::from([("meta_key".into(), "meta_value".to_string())]), + metrics: HashMap::from([("metric_key".into(), 123.45)]), + }); + + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } + + #[tokio::test] + async fn test_error_response() { + let resp = CommandResponse::ProtocolError; + let mut buf = BytesMut::new(); + let mut encoder = CommandCodec; + encoder.encode(resp, &mut buf).unwrap(); + assert!(!buf.is_empty()); + } +} diff --git a/appsec/helper-rust/src/config.rs b/appsec/helper-rust/src/config.rs new file mode 100644 index 00000000000..51bc03e52a3 --- /dev/null +++ b/appsec/helper-rust/src/config.rs @@ -0,0 +1,139 @@ +use anyhow::Result; +use std::env; +use std::ffi::OsString; +use std::fmt; +use std::os::unix::ffi::OsStringExt; +use std::path::PathBuf; +use std::str::FromStr; + +#[derive(Clone)] +pub struct Config { + pub socket_path: Vec, + pub lock_path: PathBuf, + pub log_file_path: Option, + pub log_level: log::Level, +} + +impl fmt::Debug for Config { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let socket_display = if self.socket_path.first() == Some(&0) { + let mut display = String::from("@"); + display.push_str(&String::from_utf8_lossy(&self.socket_path[1..])); + display + } else { + String::from_utf8_lossy(&self.socket_path).to_string() + }; + + f.debug_struct("Config") + .field("socket_path", &socket_display) + .field("lock_path", &self.lock_path) + .field("log_file_path", &self.log_file_path) + .field("log_level", &self.log_level) + .finish() + } +} + +impl Config { + #[cfg(target_os = "linux")] + const DEFAULT_SOCKET_PATH: &'static str = "@/ddappsec"; + #[cfg(not(target_os = "linux"))] + const DEFAULT_SOCKET_PATH: &'static str = "/tmp/ddappsec.sock"; + + const DEFAULT_LOCK_PATH: &'static str = "/tmp/ddappsec.lock"; + const DEFAULT_LOG_LEVEL: log::Level = log::Level::Info; + + /// Load configuration from environment variables + pub fn from_env() -> Result { + let mut socket_path = env::var("_DD_SIDECAR_APPSEC_SOCKET_FILE_PATH") + .unwrap_or_else(|_| Self::DEFAULT_SOCKET_PATH.to_string()) + .into_bytes(); + if socket_path.first() == Some(&b'@') { + socket_path[0] = 0u8; + } + + let lock_path = env::var("_DD_SIDECAR_APPSEC_LOCK_FILE_PATH") + .unwrap_or_else(|_| Self::DEFAULT_LOCK_PATH.to_string()) + .into(); + + let log_file_path = env::var("_DD_SIDECAR_APPSEC_LOG_FILE_PATH") + .ok() + .map(PathBuf::from); + + let log_level = env::var("_DD_SIDECAR_APPSEC_LOG_LEVEL") + .map(|s| { + if s.to_lowercase() == "warning" { + "warn".into() + } else { + s + } + }) + .ok() + .and_then(|s| log::Level::from_str(&s).ok()) + .unwrap_or(Self::DEFAULT_LOG_LEVEL); + + Ok(Config { + socket_path, + lock_path, + log_file_path, + log_level, + }) + } + + pub fn socket_path_as_path(&self) -> PathBuf { + let os = OsString::from_vec(self.socket_path.clone()); + PathBuf::from(os) + } + + pub fn is_abstract_socket(&self) -> bool { + self.socket_path.first() == Some(&b'\0') + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + + #[test] + #[serial] + fn test_config_from_env_with_defaults() { + // Clear env vars to test defaults + env::remove_var("_DD_SIDECAR_APPSEC_SOCKET_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOCK_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOG_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOG_LEVEL"); + + let config = Config::from_env().unwrap(); + + #[cfg(target_os = "linux")] + assert_eq!(config.socket_path, b"\0/ddappsec"); + #[cfg(not(target_os = "linux"))] + assert_eq!(config.socket_path, b"/tmp/ddappsec.sock"); + + assert_eq!(config.lock_path, PathBuf::from("/tmp/ddappsec.lock")); + assert_eq!(config.log_file_path, None); + assert_eq!(config.log_level, log::Level::Info); + } + + #[test] + #[serial] + fn test_config_from_env_with_custom_values() { + env::set_var("_DD_SIDECAR_APPSEC_SOCKET_FILE_PATH", "/custom/socket.sock"); + env::set_var("_DD_SIDECAR_APPSEC_LOCK_FILE_PATH", "/custom/lock.lock"); + env::set_var("_DD_SIDECAR_APPSEC_LOG_FILE_PATH", "/custom/log.log"); + env::set_var("_DD_SIDECAR_APPSEC_LOG_LEVEL", "debug"); + + let config = Config::from_env().unwrap(); + + assert_eq!(config.socket_path, b"/custom/socket.sock"); + assert_eq!(config.lock_path, PathBuf::from("/custom/lock.lock")); + assert_eq!(config.log_file_path, Some(PathBuf::from("/custom/log.log"))); + assert_eq!(config.log_level, log::Level::Debug); + + // Cleanup + env::remove_var("_DD_SIDECAR_APPSEC_SOCKET_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOCK_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOG_FILE_PATH"); + env::remove_var("_DD_SIDECAR_APPSEC_LOG_LEVEL"); + } +} diff --git a/appsec/helper-rust/src/ffi.rs b/appsec/helper-rust/src/ffi.rs new file mode 100644 index 00000000000..12a8bfadf18 --- /dev/null +++ b/appsec/helper-rust/src/ffi.rs @@ -0,0 +1,198 @@ +use std::{ + cell::UnsafeCell, + ffi::CStr, + marker::PhantomData, + ops::Deref, + sync::atomic::{AtomicBool, Ordering}, +}; + +use crate::client::log::error; + +pub mod sidecar_ffi; + +#[macro_export] +macro_rules! sidecar_symbol { + // form 1: inline function signature + ( + static $static:ident = + fn($($arg:ty),* $(, ...)? $(,)?) $(-> $ret:ty)? : + $name:ident + ) => { + type $name = unsafe extern "C" fn( + $($arg),* + $(, ...)? + ) $(-> $ret)?; + + static $static: SidecarSymbol<$name> = + SidecarSymbol::new(::std::concat!(::std::stringify!($name), "\0")); + + const _: () = { + let _: $name = $name; + }; + }; + + // form 2: type alias + ( + static $static:ident = + $ty:ty : + $name:ident + ) => { + static $static: SidecarSymbol<$ty> = + SidecarSymbol::new(unsafe {::std::ffi::CStr::from_bytes_with_nul_unchecked(::std::concat!(::std::stringify!($name), "\0").as_bytes())}); + + const _: () = { + let _: $ty = $name; + }; + }; +} + +pub struct SidecarSymbol { + // In order to implement Deref, we need to have a sort of rvalue for the function + // So do not use an AtomicPtr that we check for null to determine if we're initialized + func_ptr: UnsafeCell<*mut libc::c_void>, + initialized: AtomicBool, + name: &'static CStr, + _phantom: PhantomData, +} +impl SidecarSymbol { + pub const fn new(name: &'static CStr) -> Self { + assert!( + std::mem::size_of::() == std::mem::size_of::<*mut libc::c_void>(), + "Func must be pointer-sized" + ); + Self { + func_ptr: UnsafeCell::new(std::ptr::null_mut()), + initialized: AtomicBool::new(false), + name, + _phantom: PhantomData, + } + } + + pub fn resolve(&self) -> anyhow::Result<()> { + if self.initialized.load(Ordering::Acquire) { + return Ok(()); + } + + let func_ptr = unsafe { libc::dlsym(libc::RTLD_DEFAULT, self.name.as_ptr()) }; + if func_ptr.is_null() { + return Err(anyhow::anyhow!("Failed to resolve symbol: {:?}", self.name)); + } + unsafe { std::ptr::write(self.func_ptr.get(), func_ptr) }; + self.initialized.store(true, Ordering::Release); + Ok(()) + } + + fn get(&self) -> Option<&Func> { + if !self.initialized.load(Ordering::Acquire) { + None + } else { + Some(unsafe { &*self.func_ptr.get().cast() }) + } + } +} +unsafe impl Sync for SidecarSymbol {} +impl Deref for SidecarSymbol { + type Target = Func; + + fn deref(&self) -> &Self::Target { + match self.get() { + Some(func) => func, + None => { + error!("Symbol is not resolved, will panic: {:?}", self); + panic!("Symbol is not resolved: {:?}", self.name); + } + } + } +} +impl std::fmt::Debug for SidecarSymbol { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let initialized = self.initialized.load(Ordering::Acquire); + let mut ds = f.debug_struct("SidecarSymbol"); + ds.field("name", &self.name) + .field("name", &self.name) + .field("initialized", &initialized); + if initialized { + ds.field("func_ptr", &self.func_ptr.get()); + } + ds.finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CStr; + use std::path::PathBuf; + + fn load_library(path: &std::path::Path) -> *mut libc::c_void { + let path_cstr = std::ffi::CString::new(path.to_str().unwrap()).expect("Invalid path"); + + let handle = + unsafe { libc::dlopen(path_cstr.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) }; + + if handle.is_null() { + let error = unsafe { libc::dlerror() }; + if !error.is_null() { + let error_str = unsafe { CStr::from_ptr(error) }; + panic!("dlopen failed: {:?}", error_str); + } + panic!("dlopen failed with unknown error"); + } + + handle + } + + fn unload_library(handle: *mut libc::c_void) { + if !handle.is_null() { + unsafe { + libc::dlclose(handle); + } + } + } + + type TestAddFn = unsafe extern "C" fn(i32, i32) -> i32; + + #[test] + fn test_sidecar_symbol_resolve_and_call() { + #[cfg(target_os = "macos")] + let lib_name = "libtest_sidecar.dylib"; + #[cfg(target_os = "linux")] + let lib_name = "libtest_sidecar.so"; + let lib_path = PathBuf::from(env!("OUT_DIR")).join(lib_name); + let handle = load_library(&lib_path); + + static TEST_ADD_SYMBOL: SidecarSymbol = SidecarSymbol::new(c"test_add"); + TEST_ADD_SYMBOL + .resolve() + .expect("Failed to resolve test_add symbol"); + + let result = unsafe { TEST_ADD_SYMBOL(3, 4) }; + assert_eq!(result, 7, "test_add(3, 4) should return 7"); + + unload_library(handle); + } + + #[test] + fn test_sidecar_symbol_unresolved_symbol_fails() { + static NONEXISTENT_SYMBOL: SidecarSymbol = + SidecarSymbol::new(c"nonexistent_function_12345"); + + let result = NONEXISTENT_SYMBOL.resolve(); + assert!(result.is_err(), "Resolving nonexistent symbol should fail"); + } + + #[test] + fn test_sidecar_symbol_debug_format() { + static DEBUG_TEST_SYMBOL: SidecarSymbol = SidecarSymbol::new(c"debug_test_func"); + + let debug_str = format!("{:?}", DEBUG_TEST_SYMBOL); + assert!( + debug_str.contains("initialized: false"), + "Unresolved symbol should show initialized: false" + ); + assert!( + debug_str.contains("debug_test_func"), + "Debug output should contain the symbol name" + ); + } +} diff --git a/appsec/helper-rust/src/ffi/sidecar_ffi.rs b/appsec/helper-rust/src/ffi/sidecar_ffi.rs new file mode 100644 index 00000000000..202e11de288 --- /dev/null +++ b/appsec/helper-rust/src/ffi/sidecar_ffi.rs @@ -0,0 +1,127 @@ +/* automatically generated by rust-bindgen 0.72.1 */ + +//! Auto-generated FFI bindings from components-rs/sidecar.h +//! +//! Regenerate with: ./scripts/generate-sidecar-ffi.sh +//! +//! Only includes types/functions needed for helper-rust as telemetry sender. +#![allow(non_camel_case_types, non_upper_case_globals, dead_code)] + +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ddog_Vec_U8 { + pub ptr: *const u8, + pub len: usize, + pub capacity: usize, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ddog_Error { + pub message: ddog_Vec_U8, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ddog_Slice_CChar { + pub ptr: *const ::core::ffi::c_char, + pub len: usize, +} +pub type ddog_CharSlice = ddog_Slice_CChar; +pub const ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR: ddog_Option_Error_Tag = 0; +pub const ddog_Option_Error_Tag_DDOG_OPTION_ERROR_NONE_ERROR: ddog_Option_Error_Tag = 1; +pub type ddog_Option_Error_Tag = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Copy, Clone)] +pub struct ddog_Option_Error { + pub tag: ddog_Option_Error_Tag, + pub __bindgen_anon_1: ddog_Option_Error__bindgen_ty_1, +} +#[repr(C)] +#[derive(Copy, Clone)] +pub union ddog_Option_Error__bindgen_ty_1 { + pub __bindgen_anon_1: ddog_Option_Error__bindgen_ty_1__bindgen_ty_1, +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ddog_Option_Error__bindgen_ty_1__bindgen_ty_1 { + pub some: ddog_Error, +} +pub type ddog_MaybeError = ddog_Option_Error; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_TRACERS: ddog_MetricNamespace = 0; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_PROFILERS: ddog_MetricNamespace = 1; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_RUM: ddog_MetricNamespace = 2; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_APPSEC: ddog_MetricNamespace = 3; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_IDE_PLUGINS: ddog_MetricNamespace = 4; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_LIVE_DEBUGGER: ddog_MetricNamespace = 5; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_IAST: ddog_MetricNamespace = 6; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_GENERAL: ddog_MetricNamespace = 7; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_TELEMETRY: ddog_MetricNamespace = 8; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_APM: ddog_MetricNamespace = 9; +pub const ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_SIDECAR: ddog_MetricNamespace = 10; +pub type ddog_MetricNamespace = ::core::ffi::c_uint; +pub const ddog_MetricType_DDOG_METRIC_TYPE_GAUGE: ddog_MetricType = 0; +pub const ddog_MetricType_DDOG_METRIC_TYPE_COUNT: ddog_MetricType = 1; +pub const ddog_MetricType_DDOG_METRIC_TYPE_DISTRIBUTION: ddog_MetricType = 2; +pub type ddog_MetricType = ::core::ffi::c_uint; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct ddog_SidecarTransport { + _unused: [u8; 0], +} +pub const ddog_LogLevel_DDOG_LOG_LEVEL_ERROR: ddog_LogLevel = 0; +pub const ddog_LogLevel_DDOG_LOG_LEVEL_WARN: ddog_LogLevel = 1; +pub const ddog_LogLevel_DDOG_LOG_LEVEL_DEBUG: ddog_LogLevel = 2; +pub type ddog_LogLevel = ::core::ffi::c_uint; +unsafe extern "C" { + pub fn ddog_Error_drop(error: *mut ddog_Error); +} +unsafe extern "C" { + pub fn ddog_Error_message(error: *const ddog_Error) -> ddog_CharSlice; +} +unsafe extern "C" { + pub fn ddog_MaybeError_drop(arg1: ddog_MaybeError); +} +unsafe extern "C" { + pub fn ddog_sidecar_transport_drop(arg1: *mut ddog_SidecarTransport); +} +unsafe extern "C" { + pub fn ddog_sidecar_connect(connection: *mut *mut ddog_SidecarTransport) -> ddog_MaybeError; +} +unsafe extern "C" { + pub fn ddog_sidecar_ping(transport: *mut *mut ddog_SidecarTransport) -> ddog_MaybeError; +} +unsafe extern "C" { + pub fn ddog_sidecar_enqueue_telemetry_log( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + identifier_ffi: ddog_CharSlice, + level: ddog_LogLevel, + message_ffi: ddog_CharSlice, + stack_trace_ffi: *mut ddog_CharSlice, + tags_ffi: *mut ddog_CharSlice, + is_sensitive: bool, + ) -> ddog_MaybeError; +} +unsafe extern "C" { + pub fn ddog_sidecar_enqueue_telemetry_point( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + metric_name_ffi: ddog_CharSlice, + value: f64, + tags_ffi: *mut ddog_CharSlice, + ) -> ddog_MaybeError; +} +unsafe extern "C" { + pub fn ddog_sidecar_enqueue_telemetry_metric( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + metric_name_ffi: ddog_CharSlice, + metric_type: ddog_MetricType, + metric_namespace: ddog_MetricNamespace, + ) -> ddog_MaybeError; +} diff --git a/appsec/helper-rust/src/lib.rs b/appsec/helper-rust/src/lib.rs new file mode 100644 index 00000000000..a630677ab8f --- /dev/null +++ b/appsec/helper-rust/src/lib.rs @@ -0,0 +1,319 @@ +#![warn(clippy::extra_unused_lifetimes, clippy::needless_lifetimes)] + +use anyhow::Context; +use log::Log; +use std::path::PathBuf; +use std::sync::atomic::{AtomicPtr, Ordering}; +use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +mod client; +pub mod config; +mod ffi; +mod lock; +mod rc; +mod rc_notify; +pub mod server; +mod service; +mod telemetry; + +use config::Config; +#[cfg(target_os = "linux")] +use lock::ensure_abstract_socket_unique; +use lock::LockFile; + +static RUNTIME: AtomicPtr = const { AtomicPtr::new(std::ptr::null_mut()) }; +static CANCEL_TOKEN: AtomicPtr = const { AtomicPtr::new(std::ptr::null_mut()) }; +static SERVER_HANDLE: AtomicPtr>> = + const { AtomicPtr::new(std::ptr::null_mut()) }; + +/// C API entry point: Initialize and start the AppSec helper +/// +/// This function: +/// - Initializes logging from environment variables +/// - Loads configuration from environment variables +/// - Acquires a lock file to ensure process uniqueness +/// - Creates a tokio runtime +/// - Spawns the server task +/// - Returns immediately while the server runs in the background +/// +/// Returns 0 on success, non-zero on error +#[no_mangle] +pub extern "C" fn appsec_helper_main() -> i32 { + let config = match Config::from_env() { + Ok(cfg) => cfg, + Err(e) => { + log::error!("Failed to read configuration: {}", e); + return 1; + } + }; + + if let Err(e) = init_logging(&config.log_level, &config.log_file_path) { + eprintln!("Failed to initialize logging: {}", e); + return 1; + } + + log::info!("AppSec helper starting"); + + log::info!("Configuration: {:?}", config); + + let lock = ensure_uniqueness(&config); + if let Err(e) = lock { + log::error!("Failed to ensure uniqueness: {}", e); + return 1; + } + let lock = lock.unwrap(); + + init_waf_logging(&config); + + if let Err(e) = rc_notify::resolve_symbols() { + log::error!( + "Failed to resolve RC notify symbols: {}; will not get Remote Config updates", + e + ); + } + + if let Err(e) = telemetry::resolve_symbols() { + log::error!( + "Failed to resolve sidecar telemetry symbols: {}; telemetry logs will not be submitted", + e + ); + } + + let runtime = match Runtime::new() { + Ok(rt) => rt, + Err(e) => { + log::error!("Failed to create tokio runtime: {}", e); + return 1; + } + }; + + let cancel_token = CancellationToken::new(); + + // Spawn the server task and store its handle + let server_handle = runtime.spawn(server::run_server(config, cancel_token.clone())); + + // This should never fail, because this method is supposed to be called only once + // So the value of the atomic pointer is supposed to be null. + RUNTIME + .compare_exchange( + std::ptr::null_mut(), + Box::into_raw(Box::new(runtime)), + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + CANCEL_TOKEN + .compare_exchange( + std::ptr::null_mut(), + Box::into_raw(Box::new(cancel_token)), + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + SERVER_HANDLE + .compare_exchange( + std::ptr::null_mut(), + Box::into_raw(Box::new(server_handle)), + Ordering::Release, + Ordering::Relaxed, + ) + .unwrap(); + if let Some(lock) = lock { + std::mem::forget(lock); // don't run the Drop impl + } + + log::info!("AppSec helper started successfully"); + 0 // return immediately - runtime keeps running in background +} + +/// C API entry point: Shutdown the AppSec helper gracefully +/// +/// This function implements a three-phase shutdown: +/// 1. Signal cooperative cancellation to all tasks +/// 2. Wait a grace period for tasks to finish cleanly +/// 3. Force shutdown remaining tasks with a timeout +/// +/// Total shutdown time: 2 seconds +/// +/// Returns 0 on success +#[no_mangle] +pub extern "C" fn appsec_helper_shutdown() -> i32 { + log::info!("AppSec helper shutdown initiated"); + + let maybe_cancel_token = consume_atomic_ptr(&CANCEL_TOKEN); + match maybe_cancel_token { + Some(cancel_token) => { + cancel_token.cancel(); + log::info!("Cancellation signal sent to all tasks"); + } + None => { + log::warn!("No cancellation token in shutdown; initialization failed?"); + return 0; + } + } + + // Wait for server task to complete (with timeout) + // Poll the server handle to see if it's finished, up to 1 second + let server_handle_ptr = SERVER_HANDLE.load(Ordering::Acquire); + if server_handle_ptr.is_null() { + log::warn!("No server handle in shutdown; initialization failed?"); + return 0; + } + let server_handle = unsafe { &*server_handle_ptr }; + let start = Instant::now(); + let grace_timeout = Duration::from_millis(1000); + + while start.elapsed() < grace_timeout { + if server_handle.is_finished() { + log::info!("Server task completed gracefully in {:?}", start.elapsed()); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + + if !server_handle.is_finished() { + log::warn!("Server task did not complete within grace period"); + } + + let runtime = consume_atomic_ptr(&RUNTIME).expect("Runtime should be present"); + runtime.shutdown_timeout(Duration::from_millis(1000)); + log::info!("Runtime shutdown complete"); + + log::info!("AppSec helper shutdown complete"); + 0 +} + +fn ensure_uniqueness(config: &Config) -> anyhow::Result> { + if config.is_abstract_socket() { + #[cfg(target_os = "linux")] + if let Err(e) = ensure_abstract_socket_unique(&config.socket_path) { + anyhow::bail!("Failed to ensure uniqueness: {}", e); + } else { + Ok(None) + } + #[cfg(not(target_os = "linux"))] + { + anyhow::bail!("Abstract namespace sockets are only supported on Linux"); + } + } else { + match LockFile::acquire(config.lock_path.clone()) { + Ok(lock) => Ok(Some(lock)), + Err(e) => { + anyhow::bail!("Failed to acquire lock: {}", e); + } + } + } +} + +fn init_logging(log_level: &log::Level, log_file_path: &Option) -> anyhow::Result<()> { + use simplelog::*; + use telemetry::TelemetryAwareLogger; + + let log_level_filter = match log_level { + log::Level::Error => LevelFilter::Error, + log::Level::Warn => LevelFilter::Warn, + log::Level::Info => LevelFilter::Info, + log::Level::Debug => LevelFilter::Debug, + log::Level::Trace => LevelFilter::Trace, + }; + + let config = ConfigBuilder::new() + .set_time_format_rfc3339() + .set_thread_level(LevelFilter::Debug) + .set_target_level(LevelFilter::Debug) + .build(); + + // Create the primary logger (file or terminal) + let primary_logger: Box = if let Some(log_path) = log_file_path { + let log_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("Failed to open log file: {:?}", log_path))?; + + eprintln!("AppSec helper logging to file: {:?}", log_path); + WriteLogger::new(log_level_filter, config, log_file) + } else { + // TermLogger outputs to stderr, which works in most environments. + TermLogger::new( + log_level_filter, + config, + TerminalMode::Stderr, + ColorChoice::Auto, + ) + }; + + // Wrap with telemetry-aware logger that auto-submits error logs to telemetry + let tel_aware_logger = TelemetryAwareLogger::new(primary_logger); + + log::set_max_level(log_level_filter); + log::set_boxed_logger(Box::new(tel_aware_logger))?; + + Ok(()) +} + +fn init_waf_logging(config: &Config) { + use libddwaf::log::Level as DdwafLogLevel; + use std::ffi::CStr; + + let min_level = match config.log_level { + log::Level::Error => DdwafLogLevel::Error, + log::Level::Warn => DdwafLogLevel::Warn, + log::Level::Info => DdwafLogLevel::Warn, // intentional + log::Level::Debug => DdwafLogLevel::Debug, + log::Level::Trace => DdwafLogLevel::Trace, + }; + + unsafe { + libddwaf::log::set_log_cb( + |level: DdwafLogLevel, + function: &'static CStr, + file: &'static CStr, + line: u32, + message: &[u8]| { + let msg_str = std::str::from_utf8(message); + crate::client::log::client_log_gen!( + match level { + DdwafLogLevel::Error => log::Level::Error, + DdwafLogLevel::Warn => log::Level::Warn, + DdwafLogLevel::Info => log::Level::Debug, // intentional + DdwafLogLevel::Debug => log::Level::Trace, // intentional + DdwafLogLevel::Trace => log::Level::Trace, + _ => std::unreachable!("Invalid log level"), + }, + "{} in {:?} at {:?}:{:?}", + msg_str.unwrap_or("(invalid utf-8 in message)"), + function, + file, + line + ); + }, + min_level, + ) + }; +} + +fn consume_atomic_ptr(atomic_ptr: &AtomicPtr) -> Option> { + let ptr = atomic_ptr.load(Ordering::Acquire); + if ptr.is_null() { + return None; + } + + let res = atomic_ptr.compare_exchange( + ptr, + std::ptr::null_mut(), + Ordering::Relaxed, + Ordering::Relaxed, + ); + match res { + // SAFETY: the pointer is not null, we know it came from a box, and + // we're the only thread that managed to consume the pointer. + // There is no ABA issue because the store should with a non-null value + // happens only once, in the entrypoint. + Ok(_) => Some(unsafe { Box::from_raw(ptr) }), + Err(_) => None, + } +} diff --git a/appsec/helper-rust/src/lock.rs b/appsec/helper-rust/src/lock.rs new file mode 100644 index 00000000000..72a6e128619 --- /dev/null +++ b/appsec/helper-rust/src/lock.rs @@ -0,0 +1,178 @@ +use anyhow::{Context, Result}; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::io::AsRawFd; +use std::path::PathBuf; + +/// A lock file that ensures only one instance of the helper is running +/// +/// The lock is acquired using flock and automatically released when dropped. +pub struct LockFile { + file: File, + path: PathBuf, +} + +impl LockFile { + /// Acquire an exclusive lock on the specified file + /// + /// Locking is achieved using flock(2). + pub fn acquire(path: PathBuf) -> Result { + // Open or create the lock file + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(&path) + .with_context(|| format!("Failed to open lock file: {:?}", path))?; + + // Try to acquire exclusive lock (non-blocking) + let fd = file.as_raw_fd(); + let result = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + + if result == -1 { + let err = std::io::Error::last_os_error(); + + // EWOULDBLOCK means another process holds the lock + if err.raw_os_error() == Some(libc::EWOULDBLOCK) { + let mut pid_str = String::new(); + file.read_to_string(&mut pid_str).with_context(|| { + format!( + "Failed to acquire lock and read PID from lock file: {:?}", + path + ) + })?; + + if let Ok(pid) = pid_str.trim().parse::() { + let proc_exists = unsafe { libc::kill(pid, 0) } == 0; + + if proc_exists { + anyhow::bail!("Another helper instance is already running (PID: {})", pid); + } else { + anyhow::bail!( + "The lock file is behind held, but its written PID {} is not running", + pid + ); + } + } else { + anyhow::bail!( + "Lock file {:?} is locked and contains invalid PID: {:?}", + path, + pid_str + ); + } + } else { + return Err(err).with_context(|| "Failed to flock lock file"); + } + } + + let our_pid = unsafe { libc::getpid() }; + file.set_len(0) + .with_context(|| "Failed to truncate lock file")?; + writeln!(file, "{}", our_pid).with_context(|| "Failed to write PID to lock file")?; + file.flush().with_context(|| "Failed to flush lock file")?; + + log::info!("Acquired lock file: {:?} (PID: {})", path, our_pid); + + Ok(LockFile { file, path }) + } +} + +impl Drop for LockFile { + fn drop(&mut self) { + let res = unsafe { libc::flock(self.file.as_raw_fd(), libc::LOCK_UN) }; + if res == -1 { + log::warn!( + "Failed to release lock on file {:?}: {}", + self.path, + std::io::Error::last_os_error() + ); + } + if let Err(e) = std::fs::remove_file(&self.path) { + log::warn!("Failed to remove lock file {:?}: {}", self.path, e); + } else { + log::info!("Removed lock file: {:?}", self.path); + } + } +} + +/// Check if an abstract namespace socket is already in use by attempting to connect +/// +/// Returns Ok(()) if the socket is not in use (i.e., we can proceed). +/// Returns Err if another helper is already running on this socket. +#[cfg(target_os = "linux")] +pub fn ensure_abstract_socket_unique(socket_path: &[u8]) -> Result<()> { + use std::os::linux::net::SocketAddrExt; + use std::os::unix::net::{SocketAddr, UnixStream}; + + if socket_path.first() != Some(&b'\0') { + anyhow::bail!("Socket path is not an abstract namespace socket"); + } + + let addr = SocketAddr::from_abstract_name(&socket_path[1..])?; + match UnixStream::connect_addr(&addr) { + Ok(_) => { + anyhow::bail!( + "Another helper is already running on abstract socket {}", + String::from_utf8_lossy(&socket_path[1..]) + ); + } + Err(e) + if e.kind() == std::io::ErrorKind::ConnectionRefused + || e.kind() == std::io::ErrorKind::NotFound => + { + log::debug!( + "No helper running on abstract socket {}, proceeding", + String::from_utf8_lossy(&socket_path[1..]) + ); + Ok(()) + } + Err(e) => Err(e).with_context(|| { + format!( + "Failed to check abstract socket uniqueness for {}", + String::from_utf8_lossy(&socket_path[1..]) + ) + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[test] + fn test_lock_file_acquire_and_release() { + let lock_path = PathBuf::from("/tmp/test_ddappsec.lock"); + + let _ = fs::remove_file(&lock_path); + + { + let _lock = LockFile::acquire(lock_path.clone()).unwrap(); + assert!(lock_path.exists()); + + let content = fs::read_to_string(&lock_path).unwrap(); + let pid: i32 = content.trim().parse().unwrap(); + assert_eq!(pid, unsafe { libc::getpid() }); + } + + assert!(!lock_path.exists()); + } + + #[test] + fn test_lock_file_prevents_double_lock() { + let lock_path = PathBuf::from("/tmp/test_ddappsec2.lock"); + + // Clean up any existing lock file + let _ = fs::remove_file(&lock_path); + + let _lock1 = LockFile::acquire(lock_path.clone()).unwrap(); + + // Second lock should fail + let result = LockFile::acquire(lock_path.clone()); + assert!(result.is_err()); + + // Clean up + drop(_lock1); + assert!(!lock_path.exists()); + } +} diff --git a/appsec/helper-rust/src/rc.rs b/appsec/helper-rust/src/rc.rs new file mode 100644 index 00000000000..6d7763335a8 --- /dev/null +++ b/appsec/helper-rust/src/rc.rs @@ -0,0 +1,678 @@ +use std::{ + ffi::{CString, OsStr}, + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::ffi::OsStrExt, + }, + path::{Path, PathBuf}, + sync::atomic::{fence, AtomicU64, AtomicUsize, Ordering}, + thread, +}; + +use anyhow::Context; +use base64::{self, Engine}; + +use crate::client::log::debug; + +pub struct ConfigPoller { + reader: ConfigReader, +} +impl ConfigPoller { + pub fn new(shmem_path: &Path) -> Self { + let shmem = Shmem::new(shmem_path); + ConfigPoller { + reader: ConfigReader { shmem, last_seq: 0 }, + } + } + + pub fn poll(&mut self) -> anyhow::Result> { + let res_maybe_cfg_dir = self.reader.read(); + match res_maybe_cfg_dir { + Ok(config) => Ok(config), + Err(err) => { + if let Some(io_err) = err.downcast_ref::() { + if io_err.kind() == std::io::ErrorKind::NotFound { + debug!( + "File not found while reading remote config {:?}: {}", + self, err + ); + return Ok(None); + } + } + Err(err).with_context(|| format!("Failed to read remote config: {:?}", self)) + } + } + } +} +impl std::fmt::Debug for ConfigPoller { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConfigPoller") + .field("shmem_path", &self.reader.shmem.path) + .finish() + } +} + +struct ConfigReader { + shmem: Shmem, + last_seq: u64, +} +unsafe impl Send for ConfigReader {} +unsafe impl Sync for ConfigReader {} + +impl ConfigReader { + fn read(&mut self) -> anyhow::Result> { + debug!("Reading config from shared memory {:?}", self.shmem.path); + + self.shmem + .open() + .with_context(|| format!("Failed to open shared memory file {:?}", self.shmem.path))?; + // ensure we have at least the header mapped + let fd_size = self.shmem.fd_size().with_context(|| { + format!("Failed to get shared memory size for {:?}", self.shmem.path) + })?; + if fd_size < std::mem::size_of::() { + anyhow::bail!("Shared memory file is too small to contain the header"); + } + self.shmem + .mmap(fd_size) + .with_context(|| format!("Failed to mmap {:?}, size {}", self.shmem.path, fd_size))?; + + loop { + // busy loop... + // this implements a seqlock + // + // The writer goes like this: + // w1. seq += 1 (acq-release) + // w2. write data + // w3. seq += 1 (release) + // + // The reader does this: + // r1. read seq (acquire) + // r2. read data + // r3. fence (acquire) + // r4. read seq (relaxed) + // r5. if seq is odd or changed retry + // see https://github.com/DataDog/libdatadog/pull/831 + + let mut mem_as_header = unsafe { self.shmem.as_type::()? }; + // acquire: synchronize with release on seq increments + // If the value is even we're guaranteed to see the data written just + // before the release (or later data) + let new_seq = mem_as_header.seq.load(Ordering::Acquire); + + if new_seq & 1 == 1 { + debug!("Sequence number is odd: {}", new_seq); + thread::yield_now(); + continue; + } + + if new_seq == self.last_seq { + debug!("Sequence number did not advance: {}", new_seq); + return Ok(None); + } + + let new_size = mem_as_header.size.load(Ordering::Relaxed); + let min_mapped_size = new_size + std::mem::size_of::(); + let cur_mapped_size = self.shmem.mapped_size(); + if cur_mapped_size < min_mapped_size { + let fd_size = self.shmem.fd_size()?; + if min_mapped_size > fd_size { + anyhow::bail!( + "Shared memory file is too small relatively to \ + the declared size of the payload. File size: {}, \ + declared payload size: {} -> min file size: {}", + fd_size, + new_size, + min_mapped_size + ); + } + + // remap + self.shmem + .mmap(fd_size) + .with_context(|| "Failed to map shared memory with new size")?; + mem_as_header = unsafe { self.shmem.as_type::()? }; + } + + // TODO: this should be done with core::intrinsics::atomic_load_relaxed + let mem = unsafe { self.shmem.as_slice() }; + // new_size is payload size only (not including header). + // The writer adds a trailing zero byte for C compatibility; exclude it. + let payload_start = std::mem::size_of::(); + let payload_end = payload_start + new_size - 1; + let copied_data: Vec = mem[payload_start..payload_end].to_vec(); + + // adds a LoadLoad barrier, so the following relaxed load + // cannot be moved before the read for copied_data + fence(Ordering::Acquire); + let final_seq = mem_as_header.seq.load(Ordering::Relaxed); + if final_seq > new_seq { + debug!( + "Sequence advanced while reading: {} -> {}; trying again", + new_seq, final_seq + ); + thread::yield_now(); + continue; + } + + debug!( + "Read config from shared memory {:?}: seq {}, size {}", + self.shmem.path, new_seq, new_size + ); + + self.last_seq = new_seq; + return Ok(Some(ConfigDirectory::new(copied_data))); + } + } +} + +pub struct ConfigDirectory { + data: Vec, +} +impl ConfigDirectory { + fn new(data: Vec) -> Self { + ConfigDirectory { data } + } + + pub fn runtime_id(&self) -> anyhow::Result<&str> { + self.data.iter().position(|&b| b == b'\n').map_or_else( + || Err(anyhow::anyhow!("No LF in remote config")), + |pos| { + std::str::from_utf8(&self.data[..pos]) + .context("Invalid UTF-8 in runtime_id of remote config") + }, + ) + } + + pub fn iter(&self) -> anyhow::Result>> + '_> { + self.data.iter().position(|&b| b == b'\n').map_or_else( + || Err(anyhow::anyhow!("No LF in remote config")), + |pos| { + Ok(ConfigIter { + data: &self.data[pos + 1..], + pos: 0, + }) + }, + ) + } +} +struct ConfigIter<'a> { + data: &'a [u8], + pos: usize, +} +impl<'a> Iterator for ConfigIter<'a> { + type Item = anyhow::Result>; + + fn next(&mut self) -> Option { + if self.pos >= self.data.len() { + return None; + } + + let slice = &self.data[self.pos..]; + let end = slice.iter().position(|&b| b == b'\n'); + + match end { + Some(end) => { + self.pos += end + 1; + Some(Config::from_line(&slice[..end])) + } + None => { + self.pos = self.data.len(); + Some(Err(anyhow::anyhow!( + "Missing LF iterating remote config lines" + ))) + } + } + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Config<'a> { + shm_path: &'a Path, + rc_path: String, +} +impl<'a> Config<'a> { + pub fn rc_path(&self) -> &str { + &self.rc_path + } + + fn from_line(line: &'a [u8]) -> anyhow::Result { + // Find the first ':' + let pos = line + .iter() + .position(|&b| b == b':') + .context("Invalid config line (no colon)")?; + let shm_path = &line[..pos]; + + // Find the second ':' + let pos2 = line[pos + 1..] + .iter() + .position(|&b| b == b':') + .map(|p| p + pos + 1) + .context("Invalid config line (no second colon)")?; + + // Extract and parse limiter_idx + let limiter_idx_str = &line[pos + 1..pos2]; + let _limiter_idx = std::str::from_utf8(limiter_idx_str) + .context("Invalid UTF-8 in limiter_idx")? + .parse::() + .context("Invalid config line (limiter_idx)")?; + + // Extract and decode rc_path (URL-safe base64, no padding) + let rc_path_encoded = &line[pos2 + 1..]; + let rc_path = base64::engine::general_purpose::URL_SAFE_NO_PAD + .decode(rc_path_encoded) + .with_context(|| "Failed to decode base64 rc_path") + .and_then(|bytes| String::from_utf8(bytes).context("Invalid UTF-8 for rc_path"))?; + + Ok(Config { + shm_path: Path::new(OsStr::from_bytes(shm_path)), + rc_path, + }) + } + + pub fn read(&self) -> anyhow::Result { + let mut shmem = Shmem::new(self.shm_path); + shmem + .open() + .with_context(|| format!("Failed to open shared memory file {:?}", self.shm_path))?; + let size = shmem + .fd_size() + .with_context(|| format!("Failed to get shared memory size of {:?}", self.shm_path))?; + shmem + .mmap(size) + .with_context(|| format!("Failed to map shared memory file {:?}", self.shm_path))?; + Ok(shmem) + } + + pub fn product(&'a self) -> Product<'a> { + let s = self.rc_path.as_str(); + if let Some(remainder) = s.strip_prefix("datadog/") { + if let Some((_, remainder)) = remainder.split_once("/") { + if let Some((product, _)) = remainder.split_once("/") { + return Product(product); + } + } + } else if let Some(remainder) = s.strip_prefix("employee/") { + if let Some((product, _)) = remainder.split_once('/') { + return Product(product); + } + } + + Product("UNKNOWN") + } +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Product<'a>(pub &'a str); + +impl<'a> Product<'a> { + pub fn name(&self) -> &'a str { + self.0 + } +} + +#[derive(Debug, Clone)] +pub struct ParsedConfigKey { + pub product: String, + pub config_id: String, +} + +impl ParsedConfigKey { + pub fn from_rc_path(rc_path: &str) -> Option { + // Format: (datadog/ | employee)/// + let parts: Vec<&str> = rc_path.split('/').collect(); + + if parts.len() >= 4 && parts[0] == "datadog" { + // datadog////... + Some(ParsedConfigKey { + product: parts[2].to_ascii_lowercase(), + config_id: parts[3].to_string(), + }) + } else if parts.len() >= 3 && parts[0] == "employee" { + // employee///... + Some(ParsedConfigKey { + product: parts[1].to_ascii_lowercase(), + config_id: parts[2].to_string(), + }) + } else { + None + } + } +} + +#[repr(C)] +#[derive(Debug)] +struct ConfigDirHeaderInMem { + pub seq: AtomicU64, + pub size: AtomicUsize, +} + +pub struct Shmem { + path: PathBuf, + fd: Option, + ptr: *const u8, + size: usize, // mapped size +} + +impl Shmem { + fn new(path: &Path) -> Self { + Shmem { + path: path.to_owned(), + fd: None, + ptr: std::ptr::null(), + size: 0, + } + } + + fn open(&mut self) -> anyhow::Result<()> { + if self.fd.is_some() { + return Ok(()); + } + + debug!("Opening shared memory file {:?}", self.path); + let path_cstr = CString::new(self.path.as_os_str().as_bytes()) + .with_context(|| format!("Failed to convert path {:?} to CString", self.path))?; + + let fd = unsafe { libc::shm_open(path_cstr.as_ptr(), libc::O_RDONLY, 0) }; + if fd < 0 { + let err: anyhow::Error = std::io::Error::last_os_error().into(); + return Err(err.context("shm_open() failed")); + } + // SAFETY: fd is a valid file descriptor returned by shm_open on success + self.fd = Some(unsafe { OwnedFd::from_raw_fd(fd) }); + Ok(()) + } + + fn mmap(&mut self, size: usize) -> anyhow::Result<()> { + if self.fd.is_none() { + self.open()?; + } else { + self.unmap()?; + } + let fd = self + .fd + .as_ref() + .expect("fd must be present after open") + .as_raw_fd(); + let ptr = unsafe { + libc::mmap( + std::ptr::null_mut(), + size, + libc::PROT_READ, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if ptr == libc::MAP_FAILED { + return Err(anyhow::anyhow!( + "mmap failed: {}", + std::io::Error::last_os_error() + )); + } + self.ptr = ptr as *mut u8; + self.size = size; + debug!( + "Mapped shared memory file {:?}: size {}", + self.path, self.size + ); + Ok(()) + } + + fn unmap(&mut self) -> anyhow::Result<()> { + if self.ptr.is_null() { + return Ok(()); + } + let ret = unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.size) }; + if ret != 0 { + anyhow::bail!("munmap failed: {}", std::io::Error::last_os_error()); + } + self.ptr = std::ptr::null(); + self.size = 0; + Ok(()) + } + + /// SAFETY: this function is unsafe because the data behind the slice can in + /// principle change at any time + pub unsafe fn as_slice(&self) -> &[u8] { + if self.ptr.is_null() { + return &[]; + } + unsafe { std::slice::from_raw_parts(self.ptr, self.size) } + } + + pub unsafe fn as_type(&self) -> anyhow::Result<&'_ T> { + if self.ptr.is_null() { + anyhow::bail!("Shared memory not mapped"); + } + if self.size < std::mem::size_of::() { + anyhow::bail!( + "Shared memory too small for type. Expected at least {} bytes, got {} bytes", + std::mem::size_of::(), + self.size + ); + } + unsafe { Ok(&*(self.ptr as *const T)) } + } + + fn mapped_size(&self) -> usize { + self.size + } + + fn fd_size(&self) -> anyhow::Result { + if self.fd.is_none() { + anyhow::bail!("Shared memory file not open"); + } + let mut statbuf = std::mem::MaybeUninit::uninit(); + let fd = self + .fd + .as_ref() + .expect("logically, fd must be present") + .as_raw_fd(); + let res = unsafe { libc::fstat(fd, statbuf.as_mut_ptr()) }; + if res != 0 { + return Err(anyhow::anyhow!( + "fstat failed: {}", + std::io::Error::last_os_error() + )); + } + Ok(unsafe { statbuf.assume_init().st_size as usize }) + } +} +impl Drop for Shmem { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { + libc::munmap(self.ptr as *mut libc::c_void, self.size); + } + } + // OwnedFd (when present) will be closed automatically on drop + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::os::fd::{AsFd, AsRawFd, BorrowedFd}; + use std::os::unix::ffi::OsStrExt; + + fn shm_create_and_write(name: &str, content: &[u8]) -> anyhow::Result<()> { + let c_name = CString::new(name.as_bytes()).unwrap(); + unsafe { + // Best-effort cleanup in case it already exists + libc::shm_unlink(c_name.as_ptr()); + } + let fd = unsafe { + libc::shm_open( + c_name.as_ptr(), + libc::O_CREAT | libc::O_RDWR, + 0o600 as libc::c_uint, + ) + }; + if fd < 0 { + anyhow::bail!( + "shm_open create failed: {}", + std::io::Error::last_os_error() + ); + } + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + let result = unsafe { shm_write_via_mmap(fd.as_fd(), content) }; + result + } + + // mac os doesn't support write() directly + unsafe fn shm_write_via_mmap(fd: BorrowedFd, content: &[u8]) -> anyhow::Result<()> { + let raw_fd = fd.as_raw_fd(); + if libc::ftruncate(raw_fd, content.len() as i64) != 0 { + anyhow::bail!("ftruncate failed: {}", std::io::Error::last_os_error()); + } + let ptr = libc::mmap( + std::ptr::null_mut(), + content.len(), + libc::PROT_WRITE, + libc::MAP_SHARED, + raw_fd, + 0, + ); + if ptr == libc::MAP_FAILED { + anyhow::bail!("mmap failed: {}", std::io::Error::last_os_error()); + } + std::ptr::copy_nonoverlapping(content.as_ptr(), ptr as *mut u8, content.len()); + if libc::munmap(ptr, content.len()) != 0 { + anyhow::bail!("munmap failed: {}", std::io::Error::last_os_error()); + } + Ok(()) + } + + fn shm_create_and_write_config_dir( + name: &str, + runtime_id: &str, + lines: &[String], + ) -> anyhow::Result { + let header_size = std::mem::size_of::(); + let mut payload = Vec::new(); + payload.extend_from_slice(runtime_id.as_bytes()); + payload.push(b'\n'); + for l in lines { + payload.extend_from_slice(l.as_bytes()); + payload.push(b'\n'); + } + // Add trailing null byte like the sidecar does + payload.push(0); + // Size is payload only (sidecar convention: does not include header) + let payload_size = payload.len(); + + let c_name = CString::new(name.as_bytes()).unwrap(); + unsafe { + // Best-effort cleanup in case it already exists + libc::shm_unlink(c_name.as_ptr()); + } + let fd = unsafe { + libc::shm_open( + c_name.as_ptr(), + libc::O_CREAT | libc::O_RDWR, + 0o600 as libc::c_uint, + ) + }; + if fd < 0 { + anyhow::bail!( + "shm_open create failed: {}", + std::io::Error::last_os_error() + ); + } + let fd = unsafe { OwnedFd::from_raw_fd(fd) }; + // Build contiguous buffer: header + payload + let header = ConfigDirHeaderInMem { + seq: AtomicU64::new(2), + size: AtomicUsize::new(payload_size), + }; + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const ConfigDirHeaderInMem) as *const u8, + header_size, + ) + }; + let total_size = header_size + payload_size; + let mut buf = Vec::with_capacity(total_size); + buf.extend_from_slice(header_bytes); + buf.extend_from_slice(&payload); + + let result = unsafe { shm_write_via_mmap(fd.as_fd(), &buf) }; + result?; + Ok(payload_size) + } + + #[test] + fn test_config_poller_reads_runtime_id_and_files() -> anyhow::Result<()> { + // Setup + let outer = "/helper_rust_outer_test_cfg"; + let inner1 = "/helper_rust_inner_test_cfg_1"; + let inner2 = "/helper_rust_inner_test_cfg_2"; + + let inner1_content = b"FILE1_CONTENT"; + let inner2_content = b"FILE2_CONTENT"; + shm_create_and_write(inner1, inner1_content)?; + shm_create_and_write(inner2, inner2_content)?; + + let runtime_id = "runtime-123"; + let rc1_path = "employee/apm/config1"; + let rc2_path = "employee/profiler/config2"; + let rc1_b64 = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(rc1_path.as_bytes()); + let rc2_b64 = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(rc2_path.as_bytes()); + + let entries = vec![ + format!("{}:{}:{}", inner1, 0, rc1_b64), + format!("{}:{}:{}", inner2, 1, rc2_b64), + ]; + + shm_create_and_write_config_dir(outer, runtime_id, &entries)?; + + // Run poll() + let mut poller = ConfigPoller::new(Path::new(OsStr::from_bytes(outer.as_bytes()))); + let cfg_dir_opt = poller.poll()?; + let cfg_dir = cfg_dir_opt.context("Expected Some(ConfigDirectory) from poll")?; + + // Assertions + assert_eq!(cfg_dir.runtime_id()?, runtime_id); + + let mut got = Vec::new(); + for cfg_res in cfg_dir.iter()? { + let cfg = cfg_res?; + let shmem = cfg.read()?; + let data = unsafe { shmem.as_slice() }; + got.push((cfg.rc_path.clone(), data.to_vec())); + } + + assert_eq!(got.len(), 2); + // Order should match insertion + assert_eq!(got[0].0, rc1_path); + #[cfg(target_os = "macos")] + { + // On macOS, fstat() returns padded size (16KB min), so compare only the actual content + assert_eq!(&got[0].1[..inner1_content.len()], inner1_content); + } + #[cfg(not(target_os = "macos"))] + { + assert_eq!(got[0].1, inner1_content); + } + assert_eq!(got[1].0, rc2_path); + #[cfg(target_os = "macos")] + { + assert_eq!(&got[1].1[..inner2_content.len()], inner2_content); + } + #[cfg(not(target_os = "macos"))] + { + assert_eq!(got[1].1, inner2_content); + } + + unsafe { + let _ = libc::shm_unlink(CString::new(outer.as_bytes()).unwrap().as_ptr()); + let _ = libc::shm_unlink(CString::new(inner1.as_bytes()).unwrap().as_ptr()); + let _ = libc::shm_unlink(CString::new(inner2.as_bytes()).unwrap().as_ptr()); + } + + Ok(()) + } +} diff --git a/appsec/helper-rust/src/rc_notify.rs b/appsec/helper-rust/src/rc_notify.rs new file mode 100644 index 00000000000..42c10abdf12 --- /dev/null +++ b/appsec/helper-rust/src/rc_notify.rs @@ -0,0 +1,137 @@ +use std::ffi::{c_char, c_void, CStr, OsStr}; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::sync::atomic::{AtomicPtr, Ordering}; + +use crate::service::ServiceManager; + +type InProcNotifyFn = extern "C" fn(*const c_void, *const c_void); +type DdogRemoteConfigPathFn = extern "C" fn(*const c_void, *const c_void) -> *mut c_char; +type DdogRemoteConfigPathFreeFn = extern "C" fn(*mut c_char); + +static mut DDOG_SET_RC_NOTIFY_FN: Option)> = None; +static mut DDOG_REMOTE_CONFIG_PATH: Option = None; +static mut DDOG_REMOTE_CONFIG_PATH_FREE: Option = None; + +static SERVICE_MANAGER: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); + +extern "C" fn rc_notify_callback(invariants: *const c_void, target: *const c_void) { + let service_manager = SERVICE_MANAGER.load(Ordering::Acquire); + if service_manager.is_null() { + log::warn!("No service manager to notify of remote config updates"); + return; + } + + let path = match RemoteConfigPath::new(invariants, target) { + Ok(path) => path, + Err(e) => { + log::error!("Failed to get remote config path: {}", e); + return; + } + }; + + log::info!("Remote config updated notification for {:?}", path); + + let service_manager = unsafe { &*service_manager }; + service_manager.notify_of_rc_updates(path.as_ref()); +} + +pub fn resolve_symbols() -> Result<(), String> { + unsafe { + let set_fn = libc::dlsym(libc::RTLD_DEFAULT, c"ddog_set_rc_notify_fn".as_ptr()); + if set_fn.is_null() { + return Err("Failed to resolve ddog_set_rc_notify_fn".to_string()); + } + DDOG_SET_RC_NOTIFY_FN = Some(std::mem::transmute::< + *mut libc::c_void, + extern "C" fn(Option), + >(set_fn)); + + let path_fn = libc::dlsym(libc::RTLD_DEFAULT, c"ddog_remote_config_path".as_ptr()); + if path_fn.is_null() { + return Err("Failed to resolve ddog_remote_config_path".to_string()); + } + DDOG_REMOTE_CONFIG_PATH = Some(std::mem::transmute::< + *mut libc::c_void, + DdogRemoteConfigPathFn, + >(path_fn)); + + let path_free_fn = + libc::dlsym(libc::RTLD_DEFAULT, c"ddog_remote_config_path_free".as_ptr()); + if path_free_fn.is_null() { + return Err("Failed to resolve ddog_remote_config_path_free".to_string()); + } + DDOG_REMOTE_CONFIG_PATH_FREE = Some(std::mem::transmute::< + *mut libc::c_void, + DdogRemoteConfigPathFreeFn, + >(path_free_fn)); + } + + Ok(()) +} + +pub fn register_for_rc_notifications(service_manager: &'static ServiceManager) { + log::info!("Registering for RC update callbacks"); + + SERVICE_MANAGER.store(service_manager as *const _ as *mut _, Ordering::Release); + + if let Some(set_fn) = unsafe { DDOG_SET_RC_NOTIFY_FN } { + set_fn(Some(rc_notify_callback)); + } else { + log::warn!("ddog_set_rc_notify_fn not available, RC notifications will not work"); + } +} + +pub fn unregister_for_rc_notifications() { + log::info!("Unregistering for RC update callbacks"); + + if let Some(set_fn) = unsafe { DDOG_SET_RC_NOTIFY_FN } { + set_fn(None); + } + + SERVICE_MANAGER.store(std::ptr::null_mut(), Ordering::Release); +} + +struct RemoteConfigPath { + buf: *mut c_char, + path_free_fn: DdogRemoteConfigPathFreeFn, +} +impl RemoteConfigPath { + fn new(invariants: *const c_void, target: *const c_void) -> anyhow::Result { + let path_fn = match unsafe { DDOG_REMOTE_CONFIG_PATH } { + Some(f) => f, + None => { + return Err(anyhow::anyhow!("ddog_remote_config_path not resolved")); + } + }; + + let path_free_fn = match unsafe { DDOG_REMOTE_CONFIG_PATH_FREE } { + Some(f) => f, + None => { + return Err(anyhow::anyhow!("ddog_remote_config_path_free not resolved")); + } + }; + + let buf = path_fn(invariants, target); + if buf.is_null() { + return Err(anyhow::anyhow!("ddog_remote_config_path returned null")); + } + + Ok(Self { buf, path_free_fn }) + } +} +impl Drop for RemoteConfigPath { + fn drop(&mut self) { + (self.path_free_fn)(self.buf); + } +} +impl AsRef for RemoteConfigPath { + fn as_ref(&self) -> &Path { + unsafe { Path::new(OsStr::from_bytes(CStr::from_ptr(self.buf).to_bytes())) } + } +} +impl std::fmt::Debug for RemoteConfigPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "RemoteConfigPath {:?}", self.as_ref()) + } +} diff --git a/appsec/helper-rust/src/server.rs b/appsec/helper-rust/src/server.rs new file mode 100644 index 00000000000..1afe83eebbe --- /dev/null +++ b/appsec/helper-rust/src/server.rs @@ -0,0 +1,82 @@ +use anyhow::Context; +use tokio::net::UnixListener; +use tokio_util::future::FutureExt; +use tokio_util::sync::CancellationToken; + +use crate::client::Client; +use crate::config::Config; +use crate::rc_notify; +use crate::service::ServiceManager; +use crate::telemetry::SidecarReadyFuture; + +/// Run the Unix socket server that accepts client connections +/// +/// This function: +/// - Binds to the configured Unix socket +/// - Accepts incoming client connections +/// - Spawns a task for each client +/// - Monitors the cancellation token for shutdown +/// +/// Returns when the cancellation token is triggered +pub async fn run_server(config: Config, cancel_token: CancellationToken) -> anyhow::Result<()> { + let socket_path = config.socket_path_as_path(); + + log::info!("Starting server on socket: {:?}", socket_path); + + #[cfg(not(target_os = "linux"))] + if config.is_abstract_socket() { + anyhow::bail!("Abstract namespace sockets are only supported on Linux"); + } + + // tokio handles abstract namespace sockets on Linux + let listener = UnixListener::bind(&socket_path) + .with_context(|| format!("binding unix socket {:?}", &socket_path))?; + + log::info!("Listening for connections"); + + // Create service manager with 'static lifetime + // We leak it since it needs to live for the entire process lifetime + let service_manager: &'static ServiceManager = Box::leak(Box::new(ServiceManager::new())); + + // Register for RC update callbacks from the sidecar + rc_notify::register_for_rc_notifications(service_manager); + + // telemetry can only be submitted after the sidecar is ready, so we need to wait for it + let sidecar_ready = SidecarReadyFuture::create(); + + loop { + match listener + .accept() + .with_cancellation_token(&cancel_token) + .await + { + Some(Ok((stream, addr))) => { + log::debug!("Accepted new client {:?}", addr); + + let client = Client::new(service_manager); + let sidecar_ready = sidecar_ready.clone(); + let token = cancel_token.clone(); + + tokio::spawn(async move { client.entrypoint(stream, sidecar_ready, token).await }); + } + Some(Err(err)) => { + log::warn!("Error in accept() call: {}", err); + } + None => { + log::info!("Server received cancellation signal, shutting down"); + break; + } + } + } + + if !config.is_abstract_socket() { + if let Err(e) = std::fs::remove_file(&socket_path) { + log::warn!("Failed to remove socket file: {}", e); + } + } + + rc_notify::unregister_for_rc_notifications(); + + log::info!("Server shutdown complete"); + Ok(()) +} diff --git a/appsec/helper-rust/src/service.rs b/appsec/helper-rust/src/service.rs new file mode 100644 index 00000000000..e66fac87a36 --- /dev/null +++ b/appsec/helper-rust/src/service.rs @@ -0,0 +1,864 @@ +use anyhow::Context; +use arc_swap::ArcSwap; +use std::{ + collections::{HashMap, HashSet}, + hash::Hash, + path::PathBuf, + sync::{Arc, Mutex, Weak}, +}; + +use crate::{ + client::{ + log::{debug, error, info, warning}, + protocol, + }, + rc, + service::config_manager::AsmFeatureConfigManager, + telemetry::{ + self, SpanMetricsGenerator, SpanMetricsSubmitter, TelemetryLog, TelemetryLogSubmitter, + TelemetryLogsCollector, TelemetryLogsGenerator, TelemetryMetricSubmitter, + TelemetryMetricsCollector, TelemetryMetricsGenerator, TelemetryTags, WAF_INIT, WAF_UPDATES, + }, +}; + +mod config_manager; +mod limiter; +mod metrics; +mod sampler; +mod updateable_waf; +mod waf_diag; +mod waf_ruleset; + +// --- Public API --- + +pub struct ServiceManager { + inner: Mutex, +} + +impl ServiceManager { + pub fn new() -> Self { + ServiceManager { + inner: Mutex::new(ServiceManagerInner { + services: HashMap::new(), + last_service: None, + }), + } + } + + pub fn get_service( + &self, + config: &ServiceFixedConfig, + tel_collector: &mut dyn TelemetryMetricSubmitter, + ) -> anyhow::Result> { + let mut inner = self.inner.lock().unwrap(); + + if let Some(weak) = inner.services.get(config) { + if let Some(service) = weak.upgrade() { + inner.last_service = Some(service.clone()); + return Ok(service); + } + } + + let service = Arc::new(Service::new(config.clone(), tel_collector)?); + inner + .services + .insert(config.clone(), Arc::downgrade(&service)); + inner.last_service = Some(service.clone()); + inner.cleanup(); + + Ok(service) + } + + pub fn notify_of_rc_updates(&self, shmem_path: &std::path::Path) { + let inner = self.inner.lock().unwrap(); + + for (config, weak) in &inner.services { + if config.rem_cfg_settings.shmem_path() != Some(shmem_path) { + continue; + } + + if let Some(service) = weak.upgrade() { + drop(inner); + if let Err(e) = service.poll_and_apply_rc() { + error!( + "Failed to apply RC update for {:?} (service with config {:?}): {}", + shmem_path, service.fixed_config, e + ); + } + return; + } + } + + debug!( + "No active service found for RC update path {:?}", + shmem_path + ); + } + + #[cfg(test)] + fn service_count(&self) -> usize { + let inner = self.inner.lock().unwrap(); + inner + .services + .values() + .filter(|w| w.strong_count() > 0) + .count() + } +} + +pub struct Service { + fixed_config: ServiceFixedConfig, + limiter: limiter::Limiter, + schema_sampler: Option, // empty => always sample + // Serializes RC polling + config application. The poller lives here because + // polling and applying must be atomic (can't have two threads poll, then both apply). + rc_update_lock: Mutex, + + // ideally, these two would be updated together atomically + waf: updateable_waf::UpdateableWafInstance, + config_snapshot: ArcSwap, // config other than waf + + // Sometimes we generate logs before we even have service/env or the errors + // happen during an RC update from a sidecar notification. + // Those need to be collected and submitted later + logs_collector: TelemetryLogsCollector, + + // Track concurrent worker count (clients handling requests) + worker_count: metrics::WorkerCountState, +} + +/// Legacy span metrics/meta from WAF initialization diagnostics. +#[derive(Debug, Clone, Default)] +pub struct InitDiagnosticsLegacy { + pub rules_loaded: u32, + pub rules_failed: u32, + pub rules_errors: String, +} +impl SpanMetricsGenerator for InitDiagnosticsLegacy { + fn generate_span_metrics(&'_ self, submitter: &mut dyn SpanMetricsSubmitter) { + submitter.submit_metric(telemetry::EVENT_RULES_LOADED, self.rules_loaded as f64); + submitter.submit_metric(telemetry::EVENT_RULES_FAILED, self.rules_failed as f64); + submitter.submit_meta(telemetry::EVENT_RULES_ERRORS, self.rules_errors.clone()); + } +} + +impl Service { + pub fn fixed_config(&self) -> &ServiceFixedConfig { + &self.fixed_config + } + + pub fn telemetry_settings(&self) -> &protocol::TelemetrySettings { + &self.fixed_config.telemetry_settings + } + + pub fn configured_waf_timeout(&self) -> Option { + self.fixed_config + .waf_settings + .waf_timeout_us + .map(std::time::Duration::from_micros) + } + + pub fn new_context(&self) -> libddwaf::Context { + self.waf.current().new_context() + } + + pub fn config_snapshot(&self) -> arc_swap::Guard> { + self.config_snapshot.load() + } + + pub fn should_force_keep(&self) -> bool { + self.limiter.go_through() + } + + pub fn waf_version() -> &'static str { + libddwaf::version().to_str().unwrap_or("unknown") + } + + pub fn should_extract_schema(&self, api_sec_samp_key: u64) -> bool { + if !self.fixed_config.waf_settings.schema_extraction.enabled { + return false; + } + + if api_sec_samp_key == 0 { + return false; + } + + self.schema_sampler + .as_ref() + .is_none_or(|sampler| sampler.should_sample(api_sec_samp_key)) + } + + pub fn poll_and_apply_rc(&self) -> anyhow::Result<()> { + let mut state = self.rc_update_lock.lock().unwrap(); + + let cfg_dir = match state.poller { + Some(ref mut poller) => match poller.poll()? { + Some(cfg_dir) => cfg_dir, + None => { + debug!("No new RC configuration"); + return Ok(()); + } + }, + None => { + debug!("RC disabled for this service"); + return Ok(()); + } + }; + + self.apply_config(&mut state, cfg_dir) + } + + fn take_pending_telemetry_metrics(&self) -> impl TelemetryMetricsGenerator { + let mut state = self.rc_update_lock.lock().unwrap(); + let pending = std::mem::take(&mut state.pending_telemetry_metrics); + pending.into_generator() + } + + pub fn take_pending_init_diagnostics_legacy(&self) -> Option { + let mut state = self.rc_update_lock.lock().unwrap(); + std::mem::take(&mut state.pending_init_diagnostics_legacy) + } + + pub fn increment_worker_count(&self) { + self.worker_count.increment(); + } + + pub fn decrement_worker_count(&self) { + self.worker_count.decrement(); + } + + fn new( + fixed_config: ServiceFixedConfig, + tel_submitter: &mut dyn TelemetryMetricSubmitter, + ) -> anyhow::Result { + let waf_settings = &fixed_config.waf_settings; + let rc_settings = &fixed_config.rem_cfg_settings; + + let mut tags = TelemetryTags::new(); + tags.add("waf_version", Self::waf_version()); + + // Load WAF ruleset + let maybe_ruleset = if let Some(path) = &waf_settings.rules_file { + waf_ruleset::WafRuleset::from_file(PathBuf::from(path)) + .with_context(|| format!("Error loading WAF ruleset from file {:?}", path)) + } else { + waf_ruleset::WafRuleset::from_default_file() + .with_context(|| "Error loading WAF ruleset from default file") + }; + + let ruleset = match maybe_ruleset { + Ok(ruleset) => ruleset, + Err(e) => { + tags.add("success", "false"); + anyhow::bail!(e); + } + }; + + // Create WAF instance + let obfuscator = libddwaf::Obfuscator::new( + waf_settings.obfuscator_key_regex.as_deref(), + waf_settings.obfuscator_value_regex.as_deref(), + ); + let config = libddwaf::Config::new(obfuscator); + let mut diagnostics = + libddwaf::object::WafOwnedDefaultAllocator::::default(); + let maybe_uwafi = updateable_waf::UpdateableWafInstance::new( + ruleset.into(), + Some(&config), + Some(&mut diagnostics), + ) + .with_context(|| "Error creating UpdateableWafInstance"); + + // Extract rules version from diagnostics + let rules_version = waf_diag::extract_ruleset_version(&diagnostics); + if let Some(ref v) = rules_version { + tags.add("event_rules_version", v); + } else { + tags.add("event_rules_version", "unknown"); + } + + // Extract legacy init diagnostics + let init_diagnostics_legacy = waf_diag::extract_init_diagnostics_legacy(&diagnostics); + + // Submit waf.init metric + tags.add("success", maybe_uwafi.is_ok().to_string()); + tel_submitter.submit_metric(WAF_INIT, 1.0, tags); + let uwafi = maybe_uwafi?; + + // Initialization of remaining components + let limiter = limiter::Limiter::new(waf_settings.trace_rate_limit); + let poller = rc_settings.shmem_path().map(rc::ConfigPoller::new); + + let schema_sampler = if waf_settings.schema_extraction.enabled + && waf_settings.schema_extraction.sampling_period >= 1.0 + { + Some(sampler::SchemaSampler::new( + waf_settings.schema_extraction.sampling_period as u32, + )) + } else { + None + }; + + let asm_always_enabled = fixed_config.always_enabled; + + let service = Service { + fixed_config, + waf: uwafi, + limiter, + schema_sampler, + rc_update_lock: Mutex::new(RcUpdateState { + poller, + last_configs: HashSet::new(), + asm_feature_config_manager: AsmFeatureConfigManager::new(), + pending_telemetry_metrics: TelemetryMetricsCollector::default(), + pending_init_diagnostics_legacy: Some(init_diagnostics_legacy), + }), + config_snapshot: ArcSwap::from_pointee(ConfigSnapshot::new( + asm_always_enabled, + rules_version, + )), + logs_collector: TelemetryLogsCollector::new(), + worker_count: Default::default(), + }; + service.poll_and_apply_rc()?; + Ok(service) + } + + fn apply_config( + &self, + state: &mut RcUpdateState, + cfg_dir: rc::ConfigDirectory, + ) -> anyhow::Result<()> { + debug!("Applying config for runtime id {}", cfg_dir.runtime_id()?); + + let mut new_snapshot = (**self.config_snapshot.load()).clone(); + let mut new_configs = HashSet::new(); + let mut waf_changed = false; + let mut all_diagnostics = Vec::new(); + let mut rules_version: Option = None; + + let maybe_cfg_iter = cfg_dir.iter(); + match maybe_cfg_iter { + Ok(cfg_iter) => { + for maybe_cfg in cfg_iter { + // Handle each new/updated config + let cfg = maybe_cfg?; + let rc_path = cfg.rc_path(); + new_configs.insert(rc_path.to_string()); + + let result = self.apply_config_file( + &cfg, + rc_path, + state, + &mut all_diagnostics, + &mut rules_version, + &mut new_snapshot, + &mut waf_changed, + ); + if let Err(e) = result { + self.log_product_rc_error(rc_path, &self.logs_collector, e); + } + } + } + Err(e) => { + self.log_general_rc_error(e.context("Failed to iterate over configs")); + } + } + + for maybe_cfg in cfg_dir.iter()? { + let cfg = maybe_cfg?; + let rc_path = cfg.rc_path(); + new_configs.insert(rc_path.to_string()); + } + + // Handle removal of configs + for old_path in state.last_configs.difference(&new_configs) { + if old_path.contains("/ASM_FEATURES/") { + state.asm_feature_config_manager.remove(old_path); + } else { + let res = self.waf.remove_config(old_path); + match res { + Ok(true) => { + debug!("Removed WAF config: {}", old_path); + waf_changed = true; + } + Ok(false) => { + warning!("No WAF config found to remove: {}", old_path); + } + Err(e) => { + self.log_general_rc_error(e.context(format!( + concat!( + "Failed to remove WAF config {}; ", + "this should happen only when reading the default config" + ), + old_path + ))); + } + } + } + } + state.last_configs = new_configs; + + // Telemetry waf.config_errors metrics and telemetry logs - always process + // Use new rules_version if set, otherwise fall back to existing snapshot's version + let version = rules_version + .as_deref() + .or(new_snapshot.rules_version.as_deref()) + .unwrap_or("unknown"); + for (rc_path, diagnostics) in &all_diagnostics { + waf_diag::report_diagnostics_errors( + rc_path, + diagnostics, + version, + &mut state.pending_telemetry_metrics, + &self.logs_collector, + ); + } + + if waf_changed { + // do WAF update + let update_success = match self.waf.update() { + Ok(_) => true, + Err(e) => { + self.log_general_rc_error( + e.context("Failed to rebuild WAF after config update"), + ); + false + } + }; + + // Telemetry waf.updates + let mut tags = TelemetryTags::new(); + tags.add("waf_version", Service::waf_version()) + .add("event_rules_version", version) + .add("success", update_success.to_string()); + state + .pending_telemetry_metrics + .submit_metric(WAF_UPDATES, 1.0, tags); + } + + let asm_features = state.asm_feature_config_manager.build_final(); + + new_snapshot = new_snapshot.with_asm_features( + self.fixed_config.always_enabled, + asm_features.asm, + asm_features.auto_user_instrum, + ); + if self.fixed_config.always_enabled && !new_snapshot.asm_enabled { + info!( + "ASM_FEATURES requested that ASM be disabled, but it's \ + forced into the enabled state by configuration", + ); + } + + info!("Updating config snapshot with {:?}", new_snapshot); + self.config_snapshot.store(Arc::new(new_snapshot)); + + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + fn apply_config_file( + &self, + cfg: &rc::Config, + rc_path: &str, + state: &mut RcUpdateState, + all_diagnostics: &mut Vec<( + String, + libddwaf::object::WafOwnedDefaultAllocator, + )>, + rules_version: &mut Option, + new_snapshot: &mut ConfigSnapshot, + waf_changed: &mut bool, + ) -> anyhow::Result<()> { + let product = cfg.product(); + debug!( + "Processing config: rc_path={}, product={}", + rc_path, + product.name() + ); + match product.name() { + "ASM_FEATURES" => { + let shmem = cfg.read()?; + let data = unsafe { shmem.as_slice() }; + state + .asm_feature_config_manager + .add(rc_path.to_string(), data)?; + Ok(()) + } + "ASM_DD" | "ASM" | "ASM_DATA" => { + let shmem = cfg.read()?; + let data = unsafe { shmem.as_slice() }; + + let ruleset = waf_ruleset::WafRuleset::from_slice(data) + .with_context(|| format!("Failed to parse WAF config for {}", rc_path))?; + + let waf_obj: libddwaf::object::WafObject = ruleset.into(); + let mut diagnostics = Default::default(); + + let upd_result = + self.waf + .add_or_update_config(rc_path, &waf_obj, Some(&mut diagnostics)); + + if upd_result.is_ok() { + debug!("Added/updated WAF config: {}", rc_path); + if product.name() == "ASM_DD" { + *rules_version = waf_diag::extract_ruleset_version(&diagnostics); + *new_snapshot = new_snapshot.with_new_rules_version(rules_version.clone()); + } + *waf_changed = true; + } + + all_diagnostics.push((rc_path.to_string(), diagnostics)); + upd_result + } + _ => { + debug!("Ignoring unknown product: {:?}", product); + Ok(()) + } + } + } + + fn log_product_rc_error( + &self, + rc_path: &str, + logs_submitter: &TelemetryLogsCollector, + error: anyhow::Error, + ) { + let message = format!( + "Failed to apply config {} in service with config {:?}: {}", + rc_path, self.fixed_config, error + ); + + warning!("{}", message); + + if let Some(parsed_key) = rc::ParsedConfigKey::from_rc_path(rc_path) { + let identifier = format!("rc::{}::exception", parsed_key.product); + let mut tags = TelemetryTags::new(); + tags.add("log_type", &identifier) + .add("appsec_config_key", rc_path) + .add("rc_config_id", &parsed_key.config_id); + + logs_submitter.submit_log(TelemetryLog { + level: telemetry::LogLevel::Error, + identifier, + message, + stack_trace: Some(error.backtrace().to_string()), + tags: Some(tags), + is_sensitive: false, + }); + } else { + error!( + "Can't parse RC path: {}; no submission of telemetry log", + rc_path + ); + } + } + + fn log_general_rc_error(&self, error: anyhow::Error) { + let message = format!( + "Failed to apply config for service with config {:?}: {}", + self.fixed_config, error + ); + warning!("{}", message); + self.logs_collector.submit_log(TelemetryLog { + level: telemetry::LogLevel::Error, + identifier: "rc::client::exception".to_string(), + message, + stack_trace: Some(error.backtrace().to_string()), + tags: None, + is_sensitive: false, + }); + } +} +impl TelemetryLogsGenerator for Service { + fn generate_telemetry_logs(&'_ self, submitter: &mut dyn TelemetryLogSubmitter) { + self.logs_collector.generate_telemetry_logs(submitter); + } +} +impl TelemetryMetricsGenerator for Service { + fn generate_telemetry_metrics(&self, submitter: &mut dyn TelemetryMetricSubmitter) { + self.worker_count.generate_telemetry_metrics(submitter); + self.take_pending_telemetry_metrics() + .generate_telemetry_metrics(submitter); + } +} + +#[derive(Eq, PartialEq, Hash, Debug, Clone)] +pub struct ServiceFixedConfig { + always_enabled: bool, + waf_settings: protocol::WafSettings, + rem_cfg_settings: protocol::RemoteConfigSettings, + telemetry_settings: protocol::TelemetrySettings, +} + +impl ServiceFixedConfig { + pub fn new( + always_enabled: bool, + waf_settings: protocol::WafSettings, + rem_cfg_settings: protocol::RemoteConfigSettings, + telemetry_settings: protocol::TelemetrySettings, + ) -> Self { + ServiceFixedConfig { + always_enabled, + waf_settings, + rem_cfg_settings, + telemetry_settings, + } + } + + pub fn config_sync_settings( + &self, + ) -> ( + &protocol::RemoteConfigSettings, + &protocol::TelemetrySettings, + ) { + (&self.rem_cfg_settings, &self.telemetry_settings) + } + + pub fn new_from_config_sync(&self, args: protocol::ConfigSyncArgs) -> Option { + let new_rem_cfg_path = PathBuf::from(args.rem_cfg_path); + let new_rem_cfg_enabled = !new_rem_cfg_path.as_os_str().is_empty(); + let maybe_new_rem_cfg_path = if new_rem_cfg_enabled { + Some(new_rem_cfg_path.as_path()) + } else { + None + }; + + if maybe_new_rem_cfg_path != self.rem_cfg_settings.shmem_path() + || args.telemetry_settings != self.telemetry_settings + { + let mut new_cfg = self.clone(); + new_cfg.rem_cfg_settings = protocol::RemoteConfigSettings::new( + !new_rem_cfg_path.as_os_str().is_empty(), + new_rem_cfg_path, + ); + new_cfg.telemetry_settings = args.telemetry_settings; + Some(new_cfg) + } else { + None + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AutoUserInstrumMode { + #[default] + Undefined, + Unknown, + Disabled, + Identification, + Anonymization, +} + +impl AutoUserInstrumMode { + pub fn as_str(&self) -> &'static str { + match self { + AutoUserInstrumMode::Undefined => "undefined", + AutoUserInstrumMode::Unknown => "unknown", + AutoUserInstrumMode::Disabled => "disabled", + AutoUserInstrumMode::Identification => "identification", + AutoUserInstrumMode::Anonymization => "anonymization", + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct ConfigSnapshot { + pub asm_enabled: bool, + pub auto_user_instrum: AutoUserInstrumMode, + pub rules_version: Option, +} +impl ConfigSnapshot { + pub fn new(asm_always_enabled: bool, rules_version: Option) -> Self { + Self { + asm_enabled: asm_always_enabled, + auto_user_instrum: AutoUserInstrumMode::Undefined, + rules_version, + } + } + + pub fn with_asm_features( + &self, + asm_always_enabled: bool, + asm_enabled: bool, + auto_user_instrum: AutoUserInstrumMode, + ) -> Self { + Self { + asm_enabled: asm_always_enabled || asm_enabled, + auto_user_instrum, + rules_version: self.rules_version.clone(), + } + } + + pub fn with_new_rules_version(&self, rules_version: Option) -> Self { + Self { + asm_enabled: self.asm_enabled, + auto_user_instrum: self.auto_user_instrum, + rules_version, + } + } +} + +// --- Implementation details --- + +struct ServiceManagerInner { + services: HashMap>, + last_service: Option>, +} + +impl ServiceManagerInner { + fn cleanup(&mut self) { + self.services.retain(|_, weak| weak.strong_count() > 0); + } +} + +struct RcUpdateState { + poller: Option, + last_configs: HashSet, + asm_feature_config_manager: AsmFeatureConfigManager, + pending_telemetry_metrics: TelemetryMetricsCollector, + pending_init_diagnostics_legacy: Option, +} + +// --- Tests --- + +#[cfg(test)] +mod tests { + use super::*; + + const TEST_RULES_FILE: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/service/testdata/minimal_rules.json" + ); + + struct NoopTelemetrySubmitter; + + impl TelemetryMetricSubmitter for NoopTelemetrySubmitter { + fn submit_metric( + &mut self, + _key: crate::telemetry::MetricName, + _value: f64, + _tags: TelemetryTags, + ) { + // no-op + } + } + + fn make_config(id: u32) -> ServiceFixedConfig { + ServiceFixedConfig::new( + true, + protocol::WafSettings { + rules_file: Some(TEST_RULES_FILE.to_string()), + waf_timeout_us: Some(10000), + trace_rate_limit: 100, + obfuscator_key_regex: None, + obfuscator_value_regex: None, + schema_extraction: protocol::SchemaExtraction { + enabled: false, + sampling_period: 1.0, + }, + }, + protocol::RemoteConfigSettings::new(false, PathBuf::from(format!("/tmp/test_{}", id))), + protocol::TelemetrySettings { + service_name: "test".to_string(), + env_name: "test".to_string(), + }, + ) + } + + #[test] + fn service_manager_returns_same_service_for_same_config() { + let manager = ServiceManager::new(); + let config = make_config(1); + + let s1 = manager + .get_service(&config, &mut NoopTelemetrySubmitter) + .unwrap(); + let s2 = manager + .get_service(&config, &mut NoopTelemetrySubmitter) + .unwrap(); + + assert!(Arc::ptr_eq(&s1, &s2)); + assert_eq!(manager.service_count(), 1); + } + + #[test] + fn service_manager_returns_different_services_for_different_configs() { + let manager = ServiceManager::new(); + let config1 = make_config(1); + let config2 = make_config(2); + + let s1 = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + let s2 = manager + .get_service(&config2, &mut NoopTelemetrySubmitter) + .unwrap(); + + assert!(!Arc::ptr_eq(&s1, &s2)); + assert_eq!(manager.service_count(), 2); + } + + #[test] + fn service_manager_cleans_up_expired_services() { + let manager = ServiceManager::new(); + let config1 = make_config(1); + let config2 = make_config(2); + + let s1 = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + let _s2 = manager + .get_service(&config2, &mut NoopTelemetrySubmitter) + .unwrap(); + assert_eq!(manager.service_count(), 2); + + drop(_s2); + + // Trigger cleanup by getting another service + let _s3 = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + assert_eq!(manager.service_count(), 1); + assert!(Arc::ptr_eq(&s1, &_s3)); + } + + #[test] + fn service_manager_keeps_last_service_alive() { + let manager = ServiceManager::new(); + let config1 = make_config(1); + let config2 = make_config(2); + + let s1 = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + drop(s1); + + // s1's config should still be alive (last_service keeps it) + let s1_again = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + assert_eq!(manager.service_count(), 1); + + // Now get a different service, making it the last_service + let s2 = manager + .get_service(&config2, &mut NoopTelemetrySubmitter) + .unwrap(); + drop(s1_again); + + // Trigger cleanup - config1 should now be gone + let _ = manager + .get_service(&config2, &mut NoopTelemetrySubmitter) + .unwrap(); + assert_eq!(manager.service_count(), 1); + + // Getting config1 should create a new service + let s1_new = manager + .get_service(&config1, &mut NoopTelemetrySubmitter) + .unwrap(); + assert_eq!(manager.service_count(), 2); + drop(s2); + drop(s1_new); + } +} diff --git a/appsec/helper-rust/src/service/config_manager.rs b/appsec/helper-rust/src/service/config_manager.rs new file mode 100644 index 00000000000..2a49a2f14ed --- /dev/null +++ b/appsec/helper-rust/src/service/config_manager.rs @@ -0,0 +1,209 @@ +use std::{collections::BTreeMap, marker::PhantomData}; + +use serde::{de, Deserialize, Deserializer}; + +use crate::service::AutoUserInstrumMode; + +pub(super) type AsmFeatureConfigManager = ConfigManager; + +pub(super) struct ConfigManager> { + configs: BTreeMap, + _phantom: PhantomData, +} + +impl> ConfigManager { + pub fn new() -> Self { + Self { + configs: BTreeMap::new(), + _phantom: PhantomData, + } + } + + pub fn add(&mut self, key: String, value_json: &[u8]) -> anyhow::Result<()> { + let value = serde_json::from_slice::(value_json).map_err(|e| { + anyhow::anyhow!( + "failed to deserialize asm features config for {key} (value: {:?}): {e}", + String::from_utf8_lossy(value_json) + ) + })?; + self.configs.insert(key, value); + Ok(()) + } + + pub fn remove(&mut self, key: impl AsRef) -> bool { + self.configs.remove(key.as_ref()).is_some() + } + + pub fn build_final(&self) -> Final { + self.configs + .values() + .fold(Default::default(), |acc: Partial, x| acc.merge(x)) + .to_final() + } +} + +pub(super) trait Mergeable: serde::de::DeserializeOwned + Default { + fn merge(self, other: &Self) -> Self; + fn to_final(self) -> Final; +} + +#[derive(Default, Debug, Deserialize)] +pub(super) struct AsmFeaturesConfig { + #[serde(default)] + asm: Option, + + #[serde(default)] + auto_user_instrum: AutoUserInstrumInner, +} + +#[derive(Default, Debug, Deserialize)] +struct AsmEnabled { + #[serde(default, deserialize_with = "deserialize_bool_or_string")] + enabled: bool, +} + +#[derive(Default, Debug, Deserialize)] +struct AutoUserInstrumInner { + #[serde(default, deserialize_with = "deserialize_auto_user_instrum_mode")] + mode: AutoUserInstrumMode, +} +pub(super) struct AsmFeaturesConfigFinal { + pub asm: bool, + pub auto_user_instrum: AutoUserInstrumMode, +} + +impl Mergeable for AsmFeaturesConfig { + fn merge(self, other: &Self) -> Self { + let mode = match (other.auto_user_instrum.mode, self.auto_user_instrum.mode) { + (new, AutoUserInstrumMode::Undefined) => new, + (AutoUserInstrumMode::Undefined, old) => old, + (new, _) => new, + }; + let asm = other + .asm + .as_ref() + .map(|a| a.enabled) + .unwrap_or_else(|| self.asm.map(|a| a.enabled).unwrap_or(false)); + Self { + asm: Some(AsmEnabled { enabled: asm }), + auto_user_instrum: AutoUserInstrumInner { mode }, + } + } + + fn to_final(self) -> AsmFeaturesConfigFinal { + AsmFeaturesConfigFinal { + asm: self.asm.map(|a| a.enabled).unwrap_or(false), + auto_user_instrum: self.auto_user_instrum.mode, + } + } +} + +/// Deserializes a boolean from either a JSON boolean or a string ("true"/"false"). +fn deserialize_bool_or_string<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum BoolOrString { + Bool(bool), + String(String), + } + + match Option::::deserialize(deserializer)? { + None => Ok(false), + Some(BoolOrString::Bool(b)) => Ok(b), + Some(BoolOrString::String(s)) => match s.to_ascii_lowercase().as_str() { + "true" => Ok(true), + "false" => Ok(false), + _ => Err(de::Error::custom(format!( + "expected 'true' or 'false', got '{s}'" + ))), + }, + } +} + +/// Deserializes AutoUserInstrumMode from a case-insensitive string. +fn deserialize_auto_user_instrum_mode<'de, D>( + deserializer: D, +) -> Result +where + D: Deserializer<'de>, +{ + let s = Option::::deserialize(deserializer)?; + Ok(match s.as_deref() { + Some(s) if s.eq_ignore_ascii_case("identification") => AutoUserInstrumMode::Identification, + Some(s) if s.eq_ignore_ascii_case("anonymization") => AutoUserInstrumMode::Anonymization, + Some(s) if s.eq_ignore_ascii_case("disabled") => AutoUserInstrumMode::Disabled, + Some(_) => AutoUserInstrumMode::Unknown, + None => AutoUserInstrumMode::default(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn asm_features_deserialization() { + let cases = [ + (r#"{}"#, false, AutoUserInstrumMode::Undefined), + ( + r#"{"asm": {"enabled": true}}"#, + true, + AutoUserInstrumMode::Undefined, + ), + ( + r#"{"asm": {"enabled": false}}"#, + false, + AutoUserInstrumMode::Undefined, + ), + ( + r#"{"asm": {"enabled": "true"}}"#, + true, + AutoUserInstrumMode::Undefined, + ), + ( + r#"{"asm": {"enabled": "FALSE"}}"#, + false, + AutoUserInstrumMode::Undefined, + ), + ( + r#"{"auto_user_instrum": {"mode": "identification"}}"#, + false, + AutoUserInstrumMode::Identification, + ), + ( + r#"{"auto_user_instrum": {"mode": "ANONYMIZATION"}}"#, + false, + AutoUserInstrumMode::Anonymization, + ), + ( + r#"{"auto_user_instrum": {"mode": "disabled"}}"#, + false, + AutoUserInstrumMode::Disabled, + ), + ( + r#"{"auto_user_instrum": {"mode": "garbage"}}"#, + false, + AutoUserInstrumMode::Unknown, + ), + ]; + + for (json, expected_asm, expected_auto) in cases { + let config: AsmFeaturesConfig = serde_json::from_str(json).unwrap(); + let final_config = config.to_final(); + assert_eq!(final_config.asm, expected_asm, "asm mismatch for {json}"); + assert_eq!( + final_config.auto_user_instrum, expected_auto, + "auto_user_instrum mismatch for {json}" + ); + } + } + + #[test] + fn asm_features_invalid_asm_type() { + let result = serde_json::from_str::(r#"{"asm": {"enabled": 1}}"#); + assert!(result.is_err()); + } +} diff --git a/appsec/helper-rust/src/service/limiter.rs b/appsec/helper-rust/src/service/limiter.rs new file mode 100644 index 00000000000..3d799cdd3fd --- /dev/null +++ b/appsec/helper-rust/src/service/limiter.rs @@ -0,0 +1,92 @@ +use std::{ + mem::MaybeUninit, + sync::atomic::{AtomicU64, Ordering}, +}; + +pub(super) struct Limiter { + max_per_second: u32, + counts: AtomicU64, +} + +impl Limiter { + pub(super) fn new(max_per_second: u32) -> Self { + Limiter { + max_per_second, + counts: 0_u64.into(), + } + } + + pub(super) fn go_through(&self) -> bool { + if self.max_per_second == 0 { + return true; + } + + let mut now_ms = monotonic_time_millis() as u32; + let mut now_sec = ((now_ms / 1000) & 0xFFF) as u16; + + let mut prev = self.counts.load(Ordering::Relaxed); + loop { + let (st_sec, st_cur, st_prev) = split_u64(prev); + + let mut cur_count = st_cur; + let mut prev_count = st_prev; + + if now_sec != st_sec { + if st_sec == now_sec - 1 { + prev_count = cur_count; + cur_count = 0; + } else if st_sec > now_sec { + // we're behind the stored value + now_ms = 0; + now_sec = st_sec; + } else { + // st_sec < now_sec - 1 + prev_count = 0; + cur_count = 0; + } + } + + let windowed_count_est = cur_count + (prev_count * (1000 - (now_ms % 1000))) / 1000; + if windowed_count_est >= self.max_per_second { + return false; + } + + cur_count += 1; + let new_counts = join_u64(now_sec, prev_count, cur_count); + match self.counts.compare_exchange_weak( + prev, + new_counts, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(next_prev) => prev = next_prev, + } + } + } +} + +fn monotonic_time_millis() -> i64 { + let mut ts: MaybeUninit = MaybeUninit::uninit(); + unsafe { + let res = libc::clock_gettime(libc::CLOCK_MONOTONIC, ts.as_mut_ptr()); + if res != 0 { + panic!("clock_gettime failed: {}", std::io::Error::last_os_error()); + } + } + let ts = unsafe { ts.assume_init() }; + (ts.tv_sec * 1_000) + (ts.tv_nsec / 1_000_000) +} + +fn split_u64(c: u64) -> (u16, u32, u32) { + let cur_sec = (c >> 48) as u16; + let prev_sec_count = ((c >> 24) & 0xFF_FFFF) as u32; + let cur_sec_count = (c & 0xFF_FFFF) as u32; + (cur_sec, prev_sec_count, cur_sec_count) +} + +fn join_u64(cur_sec: u16, prev_sec_count: u32, cur_sec_count: u32) -> u64 { + ((cur_sec as u64) << 48) + | (((prev_sec_count & 0xFF_FFFF) as u64) << 24) + | ((cur_sec_count & 0xFF_FFFF) as u64) +} diff --git a/appsec/helper-rust/src/service/metrics.rs b/appsec/helper-rust/src/service/metrics.rs new file mode 100644 index 00000000000..26266c38743 --- /dev/null +++ b/appsec/helper-rust/src/service/metrics.rs @@ -0,0 +1,51 @@ +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::telemetry::{self, TelemetryMetricSubmitter, TelemetryMetricsGenerator, TelemetryTags}; + +#[derive(Debug, Default)] +pub struct WorkerCountState { + state: AtomicU64, +} + +impl WorkerCountState { + const DIRTY_BIT: u64 = 1u64 << 63; + const COUNT_MASK: u64 = !Self::DIRTY_BIT; + + #[inline] + pub fn increment(&self) { + // Ignore the Result: the closure always returns Some(_), so it will eventually succeed. + let _ = self + .state + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |s| { + let count = (s & Self::COUNT_MASK).wrapping_add(1); + Some(count | Self::DIRTY_BIT) + }); + } + + #[inline] + pub fn decrement(&self) { + let _ = self + .state + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |s| { + let count = (s & Self::COUNT_MASK).wrapping_sub(1); + Some(count | Self::DIRTY_BIT) + }); + } + + #[inline] + fn consume_dirty(&self) -> Option { + let prev = self.state.fetch_and(Self::COUNT_MASK, Ordering::Relaxed); // clears dirty bit + (prev & Self::DIRTY_BIT != 0).then(|| prev & Self::COUNT_MASK) + } +} +impl TelemetryMetricsGenerator for WorkerCountState { + fn generate_telemetry_metrics(&self, submitter: &mut dyn TelemetryMetricSubmitter) { + if let Some(count) = self.consume_dirty() { + submitter.submit_metric( + telemetry::HELPER_WORKER_COUNT, + count as f64, + TelemetryTags::new(), + ); + } + } +} diff --git a/appsec/helper-rust/src/service/sampler.rs b/appsec/helper-rust/src/service/sampler.rs new file mode 100644 index 00000000000..c5a48045441 --- /dev/null +++ b/appsec/helper-rust/src/service/sampler.rs @@ -0,0 +1,527 @@ +use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +const MAX_ITEMS: usize = 4096; +const CAPACITY: usize = 8192; + +#[derive(Clone)] +pub(super) struct SchemaSampler { + inner: Arc, +} + +impl SchemaSampler { + pub(super) fn new(sampling_period_secs: u32) -> Self { + let start_time = Instant::now(); + let now_secs = Self::current_time_secs(); + + Self { + inner: Arc::new(SchemaSamplerInner { + table: crossbeam_epoch::Atomic::new(Table::new()), + rebuild_in_progress: AtomicBool::new(false), + threshold: sampling_period_secs, + time_bias: now_secs.saturating_sub(sampling_period_secs), + _start_time: start_time, + }), + } + } + + pub(super) fn should_sample(&self, key: u64) -> bool { + let guard = crossbeam_epoch::pin(); + // Use Acquire here to ensure that the table is seen at least as + // it was when it was fully constructed. If not, garbage data could make + // us select a slot too advanced for a given key. + let table_ptr = self.inner.table.load(Ordering::Acquire, &guard); + let table = unsafe { table_ptr.as_ref().unwrap() }; + + self.hit(table, key) + } + + fn current_time_secs() -> u32 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as u32 + } + + fn hit(&self, table: &Table, number: u64) -> bool { + let now = Self::current_time_secs().wrapping_sub(self.inner.time_bias); + let report_threshold = now.wrapping_sub(self.inner.threshold); + + 'another_slot: loop { + let (entry, exists) = table.find_slot(number); + + if !exists { + let old_size = table.size.fetch_add(1, Ordering::Relaxed); + + if old_size >= MAX_ITEMS + && self + .inner + .rebuild_in_progress + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + let sampler = self.clone(); + let report_threshold_copy = report_threshold; + std::thread::spawn(move || { + sampler.rebuild_table(report_threshold_copy); + }); + } + + if old_size >= CAPACITY { + // no space to add anything + table.size.fetch_sub(1, Ordering::Relaxed); + return false; + } + + match entry + .key + .compare_exchange(0, number, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => { + let desired_data = EntryData { + last_accessed: now, + last_reported: now, + }; + + match entry.compare_exchange_data( + EntryData { + last_accessed: 0, + last_reported: 0, + }, + desired_data, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(_) => { + // though we created the entry, another thread updated the data + return false; + } + } + } + Err(exp_number) => { + table.size.fetch_sub(1, Ordering::Relaxed); + if exp_number == number { + // another thread inserted the same number + // presumably between find_slot() and the CAS only a very + // small amount of time passed + return false; + } + continue 'another_slot; + } + } + } + + let mut cur_data = entry.load_data(Ordering::Relaxed); + + if cur_data.last_reported <= report_threshold { + // potentially a hit + let desired_data = EntryData { + last_accessed: now, + last_reported: now, + }; + + loop { + match entry.compare_exchange_data( + cur_data, + desired_data, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return true, + Err(new_data) => { + // another thread just updated it + cur_data = new_data; + // was it a hit? + if cur_data.last_accessed == cur_data.last_reported { + // then this one should not be a hit + return false; + } + + // the other thread did not register a hit + // we retry if our idea of time is ahead + if cur_data.last_accessed < now { + continue; + } + // otherwise we just return false + return false; + } + } + } + } else { + // we just update the last accessed time + let desired_data = EntryData { + last_accessed: now, + last_reported: cur_data.last_reported, + }; + + loop { + if cur_data.last_accessed >= now { + // we're behind the times + return false; + } + + match entry.compare_exchange_data( + cur_data, + desired_data, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return false, + Err(new_data) => { + cur_data = new_data; + } + } + } + } + } + } + + fn rebuild_table(&self, report_threshold: u32) { + let guard = crossbeam_epoch::pin(); + let old_table_ptr = self.inner.table.load(Ordering::Acquire, &guard); + let old_table = unsafe { old_table_ptr.as_ref().unwrap() }; + + let new_table = Table::new(); + + #[derive(Clone, Copy)] + struct CopiableEntry { + key: u64, + data: EntryData, + } + + let mut entries: Vec = Vec::with_capacity(CAPACITY); + + for slot in 0..CAPACITY { + let entry = &old_table.entries[slot]; + let key = entry.key.load(Ordering::Relaxed); + let data = entry.load_data(Ordering::Relaxed); + + if key != 0 && data.last_reported >= report_threshold { + entries.push(CopiableEntry { key, data }); + } + } + + // most recent at the top + entries.sort_by(|a, b| b.data.last_accessed.cmp(&a.data.last_accessed)); + + let count = std::cmp::min(entries.len(), MAX_ITEMS * 2 / 3); + for ce in entries.into_iter().take(count) { + let (new_entry, _) = new_table.find_slot(ce.key); + new_entry.key.store(ce.key, Ordering::Relaxed); + new_entry.store_data(ce.data, Ordering::Relaxed); + } + new_table.size.store(count, Ordering::Relaxed); + + let new_table_owned = crossbeam_epoch::Owned::new(new_table); + let old_ptr = self + .inner + .table + .swap(new_table_owned, Ordering::Release, &guard); + + self.inner + .rebuild_in_progress + .store(false, Ordering::Relaxed); + + unsafe { + guard.defer_destroy(old_ptr); + } + } +} + +struct SchemaSamplerInner { + table: crossbeam_epoch::Atomic, + rebuild_in_progress: AtomicBool, + threshold: u32, + time_bias: u32, // avoid problems with wrap arounds + _start_time: Instant, +} + +struct Table { + entries: [Entry; CAPACITY + 1], + size: AtomicUsize, +} + +impl Table { + fn new() -> Self { + let entries: [Entry; CAPACITY + 1] = [(); CAPACITY + 1].map(|_| Entry::new()); + Self { + entries, + size: AtomicUsize::new(0), + } + } + + fn find_slot(&self, number: u64) -> (&Entry, bool) { + let hash = Self::hash(number); + let orig_idx = (hash as usize) % CAPACITY; + let mut idx = orig_idx; + + loop { + let entry = &self.entries[idx]; + let key = entry.key.load(Ordering::Relaxed); + + if key == number { + return (entry, true); + } else if key == 0 { + return (entry, false); + } + + idx = (idx + 1) % CAPACITY; + if idx == orig_idx { + // should not happen... but if it does, return a fake entry + return (&self.entries[CAPACITY], true); + } + } + } + + fn hash(number: u64) -> u64 { + number | 0x8000000000000000 // 0 is not a valid key + } +} + +#[repr(C, align(8))] +struct Entry { + key: AtomicU64, + data: AtomicU64, +} + +impl Entry { + fn new() -> Self { + Self { + key: AtomicU64::new(0), + data: AtomicU64::new(0), + } + } + + fn load_data(&self, ordering: Ordering) -> EntryData { + let raw = self.data.load(ordering); + EntryData { + last_accessed: (raw >> 32) as u32, + last_reported: raw as u32, + } + } + + fn store_data(&self, data: EntryData, ordering: Ordering) { + let raw = ((data.last_accessed as u64) << 32) | (data.last_reported as u64); + self.data.store(raw, ordering); + } + + fn compare_exchange_data( + &self, + current: EntryData, + new: EntryData, + success: Ordering, + failure: Ordering, + ) -> Result { + let current_raw = ((current.last_accessed as u64) << 32) | (current.last_reported as u64); + let new_raw = ((new.last_accessed as u64) << 32) | (new.last_reported as u64); + + match self + .data + .compare_exchange(current_raw, new_raw, success, failure) + { + Ok(raw) => Ok(EntryData { + last_accessed: (raw >> 32) as u32, + last_reported: raw as u32, + }), + Err(raw) => Err(EntryData { + last_accessed: (raw >> 32) as u32, + last_reported: raw as u32, + }), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C, align(8))] +struct EntryData { + last_accessed: u32, + last_reported: u32, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicU64; + use std::thread; + use std::time::Duration; + + #[test] + fn test_basic_hit_functionality() { + let sampler = SchemaSampler::new(1); + + assert!(sampler.should_sample(12345), "First hit should return true"); + assert!( + !sampler.should_sample(12345), + "Second hit should return false" + ); + + thread::sleep(Duration::from_secs(1)); + assert!( + sampler.should_sample(12345), + "After threshold, should return true" + ); + + assert!( + !sampler.should_sample(12345), + "Immediately after, should return false" + ); + } + + #[test] + fn test_multiple_entries() { + let sampler = SchemaSampler::new(1); + + assert!(sampler.should_sample(1), "Entry 1 first hit"); + assert!(sampler.should_sample(2), "Entry 2 first hit"); + assert!(sampler.should_sample(3), "Entry 3 first hit"); + + assert!(!sampler.should_sample(1), "Entry 1 second hit"); + assert!(!sampler.should_sample(2), "Entry 2 second hit"); + assert!(!sampler.should_sample(3), "Entry 3 second hit"); + + assert!(sampler.should_sample(4), "Entry 4 first hit"); + } + + #[test] + fn test_zero_key_hashes_correctly() { + let sampler = SchemaSampler::new(1); + + // Key 0 hashes to 0x8000000000000000 due to hash function + // The sampler treats it like any other key - the zero check + // is done at the Service layer + assert!( + sampler.should_sample(0), + "First hit for key 0 should return true" + ); + assert!( + !sampler.should_sample(0), + "Second hit for key 0 should return false" + ); + } + + #[test] + fn test_near_capacity_rebuild() { + let sampler = SchemaSampler::new(10); + + for i in 1..=MAX_ITEMS { + assert!( + sampler.should_sample(i as u64), + "First hit for entry {} should succeed", + i + ); + } + + let guard = crossbeam_epoch::pin(); + let table_ptr = sampler.inner.table.load(Ordering::Acquire, &guard); + let table = unsafe { table_ptr.as_ref().unwrap() }; + let size_before = table.size.load(Ordering::Relaxed); + assert_eq!(size_before, MAX_ITEMS); + drop(guard); + + assert!( + sampler.should_sample((MAX_ITEMS + 1) as u64), + "Inserting beyond MaxItems should trigger rebuild" + ); + + let deadline = std::time::Instant::now() + Duration::from_secs(2); + let mut final_size = MAX_ITEMS + 1; + while std::time::Instant::now() < deadline { + let guard = crossbeam_epoch::pin(); + let table_ptr = sampler.inner.table.load(Ordering::Acquire, &guard); + let table = unsafe { table_ptr.as_ref().unwrap() }; + final_size = table.size.load(Ordering::Relaxed); + drop(guard); + + if final_size < MAX_ITEMS { + break; + } + thread::sleep(Duration::from_millis(10)); + } + + assert!( + final_size <= MAX_ITEMS * 2 / 3, + "After rebuild, size should be reduced to ~2/3 of MaxItems, got {}", + final_size + ); + } + + #[test] + fn test_concurrent_access() { + let sampler = SchemaSampler::new(1); + let total_hits = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + + const NUM_THREADS: usize = 8; + let mut handles = vec![]; + + for _ in 0..NUM_THREADS { + let sampler = sampler.clone(); + let total_hits = Arc::clone(&total_hits); + let stop = Arc::clone(&stop); + + handles.push(thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if sampler.should_sample(1) { + total_hits.fetch_add(1, Ordering::Relaxed); + } + if sampler.should_sample(2) { + total_hits.fetch_add(1, Ordering::Relaxed); + } + if sampler.should_sample(3) { + total_hits.fetch_add(1, Ordering::Relaxed); + } + } + })); + } + + for _ in 0..5 { + thread::sleep(Duration::from_millis(500)); + } + + stop.store(true, Ordering::Relaxed); + for handle in handles { + handle.join().unwrap(); + } + + let hits = total_hits.load(Ordering::Relaxed); + assert!( + (3 * 2..=3 * 6).contains(&hits), + "Expected between 6-18 hits (3 keys * 2-6 threshold periods), got {}", + hits + ); + } + + #[test] + fn test_hash_with_high_bit() { + let sampler = SchemaSampler::new(1); + + let key_without_high_bit = 0x1234567890ABCDEF; + let key_with_high_bit = 0x9234567890ABCDEF; + + assert!(sampler.should_sample(key_without_high_bit)); + assert!(sampler.should_sample(key_with_high_bit)); + + assert!(!sampler.should_sample(key_without_high_bit)); + assert!(!sampler.should_sample(key_with_high_bit)); + } + + #[test] + fn test_sampling_disabled_when_period_zero() { + let sampler = SchemaSampler::new(0); + + assert!( + sampler.should_sample(12345), + "First hit should always return true" + ); + assert!( + sampler.should_sample(12345), + "With period=0, sampler is None, all hits return true" + ); + assert!( + sampler.should_sample(12345), + "With period=0, sampler is None, all hits return true" + ); + } +} diff --git a/appsec/helper-rust/src/service/testdata/minimal_rules.json b/appsec/helper-rust/src/service/testdata/minimal_rules.json new file mode 100644 index 00000000000..5622690400a --- /dev/null +++ b/appsec/helper-rust/src/service/testdata/minimal_rules.json @@ -0,0 +1,33 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "0.0.1" + }, + "rules": [ + { + "id": "test-001", + "name": "Test rule", + "tags": { + "type": "test", + "category": "test" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "http.client_ip" + } + ], + "data": "blocked_ips" + }, + "operator": "ip_match" + } + ], + "transformers": [], + "on_match": [ + "block" + ] + } + ] +} diff --git a/appsec/helper-rust/src/service/updateable_waf.rs b/appsec/helper-rust/src/service/updateable_waf.rs new file mode 100644 index 00000000000..afadf09ac4e --- /dev/null +++ b/appsec/helper-rust/src/service/updateable_waf.rs @@ -0,0 +1,377 @@ +use std::sync::{Arc, Mutex}; + +use arc_swap::ArcSwap; +use libddwaf::{ + object::{WafMap, WafObject, WafOwnedDefaultAllocator}, + Builder, Config, Handle, +}; + +use crate::client::log::{error, warning}; + +/// A WAF instance that can be shared (through clone()) and updated by any thread. +/// +/// This provides a thread-safe wrapper around libddwaf's `Builder` and `Handle`, +/// allowing multiple threads to use the same WAF instance while also supporting +/// runtime configuration updates. +pub struct UpdateableWafInstance { + inner: Arc, +} + +struct UpdateableWafInstanceInner { + prot_state: Mutex, + waf_handle: ArcSwap, + initial_ruleset: WafObject, +} + +struct ProtectedState { + builder: Builder, + initial_ruleset_added: bool, +} + +impl UpdateableWafInstance { + pub const INITIAL_RULESET: &'static str = "initial_ruleset"; + const OBFUSCATOR_KEY: &str = "datadog/0/ASM_DD/0/config"; + + /// Creates a new `UpdateableWafInstance` with an initial ruleset. + /// + /// # Arguments + /// * `ruleset` - The initial ruleset to load + /// * `config` - Optional WAF configuration + /// * `diagnostics` - Optional diagnostics output for ruleset loading errors/warnings + /// + /// # Errors + /// Returns an error if the builder cannot be created or the initial ruleset + /// cannot be added. + pub fn new( + ruleset: WafObject, + config: Option<&Config>, + diagnostics: Option<&mut WafOwnedDefaultAllocator>, + ) -> anyhow::Result { + let mut builder = + Builder::new(config).ok_or_else(|| anyhow::anyhow!("Failed to create WAF builder"))?; + + if !builder.add_or_update_config(Self::INITIAL_RULESET, &ruleset, diagnostics) { + return Err(anyhow::anyhow!( + "Failed to add initial ruleset (add_or_update_config returned false)" + )); + } + + let waf = builder + .build() + .ok_or_else(|| anyhow::anyhow!("Failed to build WAF instance"))?; + + Ok(Self { + inner: Arc::new(UpdateableWafInstanceInner { + prot_state: Mutex::new(ProtectedState { + builder, + initial_ruleset_added: true, + }), + waf_handle: ArcSwap::from_pointee(waf), + initial_ruleset: ruleset, + }), + }) + } + + /// Returns a reference to the current WAF handle. + /// + /// This can be used to create new contexts for processing requests. + #[must_use] + pub fn current(&self) -> Arc { + self.inner.waf_handle.load_full() + } + + /// Adds or updates a configuration at the specified path. + /// + /// This does not automatically rebuild the WAF instance. Call [`Self::update`] + /// to apply the changes. + /// + /// # Arguments + /// * `path` - The logical path for this configuration + /// * `ruleset` - The configuration/ruleset data + /// * `diagnostics` - Optional diagnostics output + /// + /// # Returns + /// `true` if the configuration was successfully added/updated, `false` otherwise. + pub fn add_or_update_config( + &self, + path: &str, + ruleset: &impl AsRef, + diagnostics: Option<&mut WafOwnedDefaultAllocator>, + ) -> anyhow::Result<()> { + let mut guard = self.inner.prot_state.lock().unwrap(); + + if guard.initial_ruleset_added && path.contains("/ASM_DD/") { + guard.initial_ruleset_added = false; + let res = guard.builder.remove_config(Self::INITIAL_RULESET); + if !res { + warning!("Failed to remove initial ruleset: probably not present, but we it should have been"); + } + } + + let res = guard + .builder + .add_or_update_config(path, ruleset, diagnostics); + if !res { + anyhow::bail!("Failed to add/update config {path}"); + } + Ok(()) + } + + /// Removes a configuration at the specified path. + /// + /// This does not automatically rebuild the WAF instance. Call [`Self::update`] + /// to apply the changes. + /// + /// # Returns + /// `true` if a configuration was removed, `false` if no configuration existed at that path. + /// + /// # Note + /// If the last ASM_DD config is removed, the initial ruleset will be added back. + /// Consequently, when doing an update, first add the new ASM_DD config, and only + /// then remove the old one. + pub fn remove_config(&self, path: &str) -> anyhow::Result { + let mut guard = self.inner.prot_state.lock().unwrap(); + let removed = guard.builder.remove_config(path); + if removed && path.contains("/ASM_DD/") && !guard.initial_ruleset_added { + let has_other_asm_dd = Self::has_asm_dd_configs(&mut guard.builder); + if !has_other_asm_dd { + guard.initial_ruleset_added = true; + let res = guard.builder.add_or_update_config( + Self::INITIAL_RULESET, + &self.inner.initial_ruleset, + None, + ); + if res { + log::debug!("Restored initial ruleset after removing last ASM_DD config"); + } else { + anyhow::bail!("Failed to add initial ruleset after removing ASM_DD config"); + } + } + } + Ok(removed) + } + + /// Returns the number of configuration paths currently loaded. + /// + /// # Arguments + /// * `filter` - Optional regex filter to count only matching paths + #[must_use] + #[cfg(test)] + pub fn config_paths_count(&self, filter: Option<&str>) -> u32 { + let mut guard = self.inner.prot_state.lock().unwrap(); + guard.builder.config_paths_count(filter) + } + + /// Returns the configuration paths currently loaded. + /// + /// # Arguments + /// * `filter` - Optional regex filter to return only matching paths + #[must_use] + #[cfg(test)] + pub fn config_paths( + &self, + filter: Option<&str>, + ) -> WafOwnedDefaultAllocator { + let mut guard = self.inner.prot_state.lock().unwrap(); + guard.builder.config_paths(filter) + } + + fn has_asm_dd_configs(builder: &mut Builder) -> bool { + let paths = builder.config_paths(Some("/ASM_DD/")); + for path in paths.iter() { + if let Some(path_str) = path.to_str() { + if path_str != Self::OBFUSCATOR_KEY { + return true; + } + } + } + false + } + + /// Rebuilds the WAF instance with the current configuration. + /// + /// This applies any changes made via [`Self::add_or_update_config`] or + /// [`Self::remove_config`] and atomically swaps the old instance with the new one. + /// + /// # Errors + /// Returns an error if the WAF instance cannot be built with the current configuration. + pub fn update(&self) -> anyhow::Result> { + let mut guard = self.inner.prot_state.lock().unwrap(); + let new_instance = guard + .builder + .build() + .ok_or_else(|| anyhow::anyhow!("Failed to build WAF instance"))?; + let new_instance = Arc::new(new_instance); + self.inner.waf_handle.store(new_instance.clone()); + Ok(new_instance) + } +} + +impl Clone for UpdateableWafInstance { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +#[cfg(test)] +mod tests { + use crate::service::updateable_waf::UpdateableWafInstance; + use libddwaf::object::WafObject; + use libddwaf::{waf_map, Config, RunResult, RunnableContext}; + use std::{ + sync::{ + atomic::{AtomicBool, Ordering::Relaxed}, + Arc, + }, + thread::{self, sleep}, + time::Duration, + }; + + const ARACHNI_RULE: &str = r#" +{ + "rules" : [ + { + "conditions" : [ + { + "operator" : "match_regex", + "parameters" : { + "inputs" : [ + { + "address" : "server.request.headers.no_cookies", + "key_path" : [ + "user-agent" + ] + }, + { + "address" : "server.request.body" + } + ], + "regex" : "Arachni" + } + } + ], + "id" : "arachni_rule", + "name" : "Block with default action", + "on_match" : [ + "block" + ], + "tags" : { + "category" : "attack_attempt", + "type" : "security_scanner" + } + } + ], + "version" : "2.1" +} +"#; + + const DISABLE_ARACHNI_RULE_PATH: &str = "disable_arachni"; + const DISABLE_ARACHNI_RULE: &str = r#" +{ + "rules_override": [ + { + "rules_target": [ + { + "rule_id": "arachni_rule" + } + ], + "enabled": false + } + ] +} +"#; + + #[test] + fn threaded_updateable_waf_instance() { + let ruleset: WafObject = serde_json::from_str(ARACHNI_RULE).unwrap(); + let upd_waf = UpdateableWafInstance::new(ruleset, Some(&Config::default()), None).unwrap(); + + // add a second rule because it's forbidden to have no rules + let ruleset2: WafObject = serde_json::from_str( + &ARACHNI_RULE + .replace("Arachni", "Inhcara") + .replace("arachni_rule", "inhcara_rule"), + ) + .unwrap(); + upd_waf.add_or_update_config("2nd rule", &ruleset2, None); + + assert_eq!(upd_waf.config_paths_count(Some("2nd rule")), 1); + let paths = upd_waf.config_paths(Some("2nd rule")); + assert_eq!(paths.len(), 1); + + let update_thread = std::thread::spawn({ + let upd_waf_copy = upd_waf.clone(); + let disable_ruleset: WafObject = serde_json::from_str(DISABLE_ARACHNI_RULE).unwrap(); + move || { + let mut disable_next = true; + for _ in 0..10 { + sleep(Duration::from_millis(100)); + if disable_next { + let res = upd_waf_copy.add_or_update_config( + DISABLE_ARACHNI_RULE_PATH, + &disable_ruleset, + None, + ); + if res.is_err() { + panic!("add_or_update_config failed"); + } + println!("disable"); + } else { + upd_waf_copy + .remove_config(DISABLE_ARACHNI_RULE_PATH) + .unwrap(); + println!("enable"); + } + upd_waf_copy.update().expect("update did not succeed"); + disable_next = !disable_next; + } + } + }); + + let data = Arc::new(waf_map!(( + "server.request.headers.no_cookies", + waf_map!(("user-agent", "Arachni")) + ))); + + let stop_signal = &*Box::leak(Box::new(AtomicBool::new(false))); + let t: Vec<_> = (0..2) + .map(|_| { + std::thread::spawn({ + let upd_waf_copy = upd_waf.clone(); + let data_copy = data.clone(); + let mut matches = 0u64; + let mut non_matches = 0u64; + move || { + while !stop_signal.load(Relaxed) { + let cur_instance = upd_waf_copy.current(); + println!("address of instance: {:p}", Arc::as_ptr(&cur_instance)); + let mut ctx = cur_instance.new_context(); + let res = + ctx.run(data_copy.as_ref().clone(), Duration::from_millis(500)); + match res { + Ok(RunResult::Match(_)) => { + matches += 1; + } + _ => non_matches += 1, + }; + thread::sleep(Duration::from_millis(20)) + } + (matches, non_matches) + } + }) + }) + .collect::>(); + + update_thread.join().unwrap(); + stop_signal.store(true, Relaxed); + + for jh in t { + let (matches, non_matches) = jh.join().unwrap(); + println!("positive: {matches}, negative: {non_matches}"); + assert!(matches > 10); + assert!(non_matches > 10); + } + } +} diff --git a/appsec/helper-rust/src/service/waf_diag.rs b/appsec/helper-rust/src/service/waf_diag.rs new file mode 100644 index 00000000000..091f289177c --- /dev/null +++ b/appsec/helper-rust/src/service/waf_diag.rs @@ -0,0 +1,312 @@ +use crate::{ + client::log::{debug, warning}, + rc::ParsedConfigKey, + telemetry::{ + LogLevel, TelemetryLog, TelemetryLogsCollector, TelemetryMetricSubmitter, TelemetryTags, + WAF_CONFIG_ERRORS, + }, +}; + +use super::{InitDiagnosticsLegacy, Service}; + +const DIAGNOSTIC_KEYS: &[&str] = &[ + "actions", + "custom_rules", + "exclusion_data", + "exclusions", + "processors", + "rules", + "rules_data", + "rules_override", + "scanners", +]; + +pub fn report_diagnostics_errors( + rc_path: &str, + diagnostics: &libddwaf::object::WafOwnedDefaultAllocator, + rules_version: &str, + metric_submitter: &mut impl TelemetryMetricSubmitter, + log_submitter: &TelemetryLogsCollector, +) { + use libddwaf::object::WafObjectType; + + let maybe_parsed_key = ParsedConfigKey::from_rc_path(rc_path); + let parsed_key = match maybe_parsed_key { + Some(parsed_key) => { + debug!( + "Processing diagnostics for {}: {} keys", + rc_path, + diagnostics.len() + ); + parsed_key + } + None => { + warning!("Failed to parse config key for {}", rc_path,); + return; + } + }; + + for &config_key in DIAGNOSTIC_KEYS { + let Some(keyed) = diagnostics.get_str(config_key) else { + continue; + }; + let value = keyed.value(); + if value.object_type() != WafObjectType::Map { + warning!( + "Diagnostic key {} for {} is not a map, skipping", + config_key, + rc_path + ); + continue; + } + let map = value + .as_type::() + .expect("type check"); + if map.is_empty() { + continue; + } + + debug!( + "Diagnostic {} for {} has {} entries", + config_key, + rc_path, + map.len() + ); + for kv in map.iter() { + let key_str = kv + .key() + .as_type::() + .and_then(|s| s.as_str().ok()); + debug!( + " - key: {:?}, type: {:?}", + key_str, + kv.value().object_type() + ); + } + + let mut tags = TelemetryTags::new(); + tags.add("waf_version", Service::waf_version()) + .add("event_rules_version", rules_version) + .add("config_key", config_key); + + if let Some(error_keyed) = map.get_str("error") { + if error_keyed.value().object_type() == WafObjectType::String { + tags.add("scope", "top-level"); + metric_submitter.submit_metric(WAF_CONFIG_ERRORS, 1.0, tags); + + let message = error_keyed + .value() + .as_type::() + .and_then(|s| s.as_str().ok().map(|s| s.to_string())) + .unwrap_or_default(); + submit_diagnostic_log( + log_submitter, + &parsed_key, + config_key, + "error", + LogLevel::Error, + message, + ); + continue; + } + } + + if let Some(errors_keyed) = map.get_str("errors") { + let errors = errors_keyed.value(); + if errors.object_type() == WafObjectType::Map { + let errors_map = errors + .as_type::() + .expect("type check"); + if !errors_map.is_empty() { + let mut error_count: u64 = 0; + for kv in errors_map.iter() { + let arr = kv.value(); + if arr.object_type() == WafObjectType::Array { + error_count += arr + .as_type::() + .expect("type check") + .len() as u64; + } + } + + if error_count > 0 { + let mut error_tags = tags.clone(); + error_tags.add("scope", "item"); + metric_submitter.submit_metric( + WAF_CONFIG_ERRORS, + error_count as f64, + error_tags, + ); + + let message = waf_object_to_json(errors); + submit_diagnostic_log( + log_submitter, + &parsed_key, + config_key, + "errors", + LogLevel::Error, + message, + ); + } + } + } + } + + if let Some(warnings_keyed) = map.get_str("warnings") { + let warnings = warnings_keyed.value(); + if warnings.object_type() == WafObjectType::Map { + let warnings_map = warnings + .as_type::() + .expect("type check"); + if !warnings_map.is_empty() { + let message = waf_object_to_json(warnings); + submit_diagnostic_log( + log_submitter, + &parsed_key, + config_key, + "warnings", + LogLevel::Warn, + message, + ); + } + } + } + } +} + +pub fn extract_ruleset_version( + diagnostics: &libddwaf::object::WafOwnedDefaultAllocator, +) -> Option { + use libddwaf::object::WafObjectType; + + let version_keyed = diagnostics.get_str("ruleset_version")?; + let version = version_keyed.value(); + if version.object_type() != WafObjectType::String { + return None; + } + version + .as_type::() + .and_then(|s| s.as_str().ok()) + .map(|s| s.to_string()) +} + +pub fn extract_init_diagnostics_legacy( + diagnostics: &libddwaf::object::WafOwnedDefaultAllocator, +) -> InitDiagnosticsLegacy { + use libddwaf::object::WafObjectType; + + let mut result = InitDiagnosticsLegacy::default(); + + let Some(rules_keyed) = diagnostics.get_str("rules") else { + return result; + }; + let rules = rules_keyed.value(); + if rules.object_type() != WafObjectType::Map { + return result; + } + let rules_map = rules + .as_type::() + .expect("type check"); + + if let Some(loaded_keyed) = rules_map.get_str("loaded") { + let loaded = loaded_keyed.value(); + if loaded.object_type() == WafObjectType::Array { + result.rules_loaded = loaded + .as_type::() + .expect("type check") + .len() as u32; + } + } + + if let Some(failed_keyed) = rules_map.get_str("failed") { + let failed = failed_keyed.value(); + if failed.object_type() == WafObjectType::Array { + result.rules_failed = failed + .as_type::() + .expect("type check") + .len() as u32; + } + } + + if let Some(errors_keyed) = rules_map.get_str("errors") { + let errors = errors_keyed.value(); + if errors.object_type() == WafObjectType::Map { + result.rules_errors = + serde_json::to_string(errors).unwrap_or_else(|_| "{}".to_string()); + } + } + + if result.rules_errors.is_empty() { + result.rules_errors = "{}".to_string(); + } + + result +} + +fn submit_diagnostic_log( + log_submitter: &TelemetryLogsCollector, + parsed_key: &ParsedConfigKey, + config_key: &str, + suffix: &str, + level: LogLevel, + message: String, +) { + let log_type = format!("rc::{}::diagnostic", parsed_key.product); + let identifier = format!("{}::{}", log_type, suffix); + let mut log_tags = TelemetryTags::new(); + log_tags + .add("log_type", &log_type) + .add("appsec_config_key", config_key) + .add("rc_config_id", &parsed_key.config_id); + log_submitter.submit_log(TelemetryLog { + level, + identifier, + message, + stack_trace: None, + tags: Some(log_tags), + is_sensitive: false, + }); +} + +fn waf_object_to_json(obj: &libddwaf::object::WafObject) -> String { + use libddwaf::object::WafObjectType; + use serde_json::{Map, Value}; + + fn convert(obj: &libddwaf::object::WafObject) -> Value { + match obj.object_type() { + WafObjectType::Map => { + let map = obj + .as_type::() + .expect("type check"); + let mut json_map = Map::new(); + for kv in map.iter() { + let key = kv + .key() + .as_type::() + .and_then(|s| s.as_str().ok()) + .unwrap_or("") + .to_string(); + json_map.insert(key, convert(kv.value())); + } + Value::Object(json_map) + } + WafObjectType::Array => { + let arr = obj + .as_type::() + .expect("type check"); + let json_arr: Vec = arr.iter().map(convert).collect(); + Value::Array(json_arr) + } + WafObjectType::String => { + let s = obj + .as_type::() + .and_then(|s| s.as_str().ok()) + .unwrap_or(""); + Value::String(s.to_string()) + } + _ => Value::Null, + } + } + + serde_json::to_string(&convert(obj)).unwrap_or_else(|_| "{}".to_string()) +} diff --git a/appsec/helper-rust/src/service/waf_ruleset.rs b/appsec/helper-rust/src/service/waf_ruleset.rs new file mode 100644 index 00000000000..3bc11848ca3 --- /dev/null +++ b/appsec/helper-rust/src/service/waf_ruleset.rs @@ -0,0 +1,132 @@ +use crate::client::log; +use std::{ + fs::File, + io::{BufRead, BufReader, Read, Seek}, + path::{Path, PathBuf}, +}; + +use anyhow::{anyhow, Context}; + +pub struct WafRuleset { + doc: libddwaf::object::WafObject, + rules_version: Option, +} + +impl WafRuleset { + pub fn new(doc: libddwaf::object::WafObject, rules_version: Option) -> Self { + WafRuleset { doc, rules_version } + } + + pub fn from_file>(path: P) -> anyhow::Result { + let mut reader = BufReader::new(File::open(&path)?); + + let rules_version = extract_rules_version(&mut reader); + reader.rewind()?; + + let doc: libddwaf::object::WafObject = + serde_json::from_reader(reader).with_context(|| { + format!( + "Error deserializing ddwaf_object data from json file {:?}", + path.as_ref() + ) + })?; + + log::info!("Loaded WAF ruleset from {:?}", path.as_ref()); + + Ok(WafRuleset::new(doc, rules_version)) + } + + pub fn from_default_file() -> anyhow::Result { + let file = get_default_rules_file()?; + WafRuleset::from_file(&file) + } + + pub fn from_slice(slice: &[u8]) -> anyhow::Result { + let rules_version = extract_rules_version(slice); + let doc: libddwaf::object::WafObject = serde_json::from_slice(slice) + .with_context(|| "Error deserializing ddwaf_object data from json slice")?; + Ok(WafRuleset::new(doc, rules_version)) + } + + pub fn rules_version(&self) -> Option<&str> { + self.rules_version.as_deref() + } +} +impl From for libddwaf::object::WafObject { + fn from(val: WafRuleset) -> Self { + val.doc + } +} + +fn extract_rules_version(reader: R) -> Option { + #[derive(serde::Deserialize)] + struct RulesetMetadata { + metadata: Option, + } + #[derive(serde::Deserialize)] + struct Metadata { + rules_version: Option, + } + + let parsed: RulesetMetadata = serde_json::from_reader(reader).ok()?; + parsed.metadata?.rules_version +} + +fn get_default_rules_file() -> anyhow::Result { + let helper_path = get_helper_path(); + + let base_path: PathBuf = if let Ok(helper_path) = helper_path { + helper_path + .parent() + .ok_or(anyhow!("No parent for {:?}", helper_path))? + .to_path_buf() + } else { + get_self_path().with_context(|| "Could find neither lib path nor self exe path")? + }; + + let file = base_path.join("../etc/recommended.json"); + if file.exists() { + return Ok(file); + } + + let file_legacy = base_path.join("../etc/dd-appsec/recommended.json"); + if file_legacy.exists() { + return Ok(file_legacy); + } + + Err(anyhow!( + "Could not find recommended.json in either ../etc/ or ../etc/dd-appsec/" + )) +} + +fn get_helper_path() -> anyhow::Result { + // Match both libddappsec-helper.so (C++ helper) and + // libddappsec-helper-rust.so (Rust helper) + const LIBNAME_PREFIX: &str = "/libddappsec-helper"; + const MAPS_PATH: &str = "/proc/self/maps"; + + let file = File::open(MAPS_PATH)?; + let reader = BufReader::new(file); + + for line in reader.lines() { + let line = line?; + if line.contains(LIBNAME_PREFIX) { + if let Some(pos) = line.find('/') { + return Ok(PathBuf::from(&line[pos..])); + } else { + return Err(anyhow!("Should not happen")); + } + } + } + + Err(anyhow!( + "Could not find libddappsec-helper*.so in /proc/self/maps" + )) +} + +pub fn get_self_path() -> anyhow::Result { + const SELF_EXE: &str = "/proc/self/exe"; + + let path = std::fs::read_link(SELF_EXE)?; + Ok(path) +} diff --git a/appsec/helper-rust/src/telemetry.rs b/appsec/helper-rust/src/telemetry.rs new file mode 100644 index 00000000000..29e1cd23ab3 --- /dev/null +++ b/appsec/helper-rust/src/telemetry.rs @@ -0,0 +1,276 @@ +use crate::client::protocol::{SidecarSettings, TelemetrySettings}; +use crate::ffi::sidecar_ffi::{ + ddog_MetricType, ddog_MetricType_DDOG_METRIC_TYPE_COUNT, ddog_MetricType_DDOG_METRIC_TYPE_GAUGE, +}; +use std::cell::Cell; +use std::collections::HashMap; +use std::sync::Mutex; + +pub mod error_tel_ctx; +mod sidecar; +mod tel_aware_logger; + +pub use sidecar::{resolve_symbols, TelemetrySidecarLogSubmitter, TelemetrySidecarMetricSubmitter}; +pub use sidecar::{SidecarReadyFuture, SidecarStatus}; +pub use tel_aware_logger::TelemetryAwareLogger; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct MetricName(pub &'static str); +#[derive(Debug, Clone, Copy)] +pub struct SpanMetricName(pub &'static str); +#[derive(Debug, Clone, Copy)] +pub struct SpanMetaName(pub &'static str); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub enum LogLevel { + Error, + Warn, + #[allow(dead_code)] + Debug, +} + +pub trait SpanMetricsSubmitter { + fn submit_metric(&mut self, key: SpanMetricName, value: f64); + fn submit_meta(&mut self, key: SpanMetaName, value: String); + fn submit_meta_dyn_key(&mut self, key: String, value: String); + fn submit_metric_dyn_key(&mut self, key: String, value: f64); +} +pub trait SpanMetricsGenerator { + fn generate_span_metrics(&'_ self, submitter: &mut dyn SpanMetricsSubmitter); +} + +pub trait SpanMetaGenerator { + fn generate_meta(&'_ self, submitter: &mut dyn SpanMetricsSubmitter); +} + +pub trait TelemetryMetricSubmitter { + fn submit_metric(&mut self, key: MetricName, value: f64, tags: TelemetryTags); +} + +pub trait TelemetryMetricsGenerator { + fn generate_telemetry_metrics(&self, submitter: &mut dyn TelemetryMetricSubmitter); +} + +pub trait TelemetryLogSubmitter { + fn submit_log(&mut self, log: TelemetryLog); +} +pub trait TelemetryLogsGenerator { + fn generate_telemetry_logs(&'_ self, submitter: &mut dyn TelemetryLogSubmitter); +} + +#[derive(Default, Debug, PartialEq, Eq, Clone)] +pub struct TelemetryTags { + data: String, +} +impl TelemetryTags { + pub fn new() -> Self { + Self::default() + } + + pub fn add(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Self { + if !self.data.is_empty() { + self.data.push(','); + } + self.data.push_str(key.as_ref()); + self.data.push(':'); + self.data.push_str(value.as_ref()); + self + } + + pub fn into_string(self) -> String { + self.data + } +} +impl From for String { + fn from(tags: TelemetryTags) -> String { + tags.data + } +} + +pub const WAF_INIT: MetricName = MetricName("waf.init"); +pub const WAF_UPDATES: MetricName = MetricName("waf.updates"); +pub const WAF_REQUESTS: MetricName = MetricName("waf.requests"); +pub const WAF_CONFIG_ERRORS: MetricName = MetricName("waf.config_errors"); +pub const RASP_RULE_EVAL: MetricName = MetricName("rasp.rule.eval"); +pub const RASP_RULE_MATCH: MetricName = MetricName("rasp.rule.match"); +pub const RASP_TIMEOUT: MetricName = MetricName("rasp.timeout"); +pub const HELPER_WORKER_COUNT: MetricName = MetricName("helper.service_worker_count"); + +#[derive(Debug, Clone, Copy)] +pub struct KnownMetric { + pub name: MetricName, + pub metric_type: ddog_MetricType, +} +pub const KNOWN_METRICS: &[KnownMetric] = &[ + KnownMetric { + name: WAF_REQUESTS, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: WAF_UPDATES, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: WAF_INIT, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: WAF_CONFIG_ERRORS, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: RASP_TIMEOUT, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: RASP_RULE_MATCH, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: RASP_RULE_EVAL, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_COUNT, + }, + KnownMetric { + name: HELPER_WORKER_COUNT, + metric_type: ddog_MetricType_DDOG_METRIC_TYPE_GAUGE, + }, +]; + +pub fn register_known_metrics( + sidecar_settings: &SidecarSettings, + telemetry_settings: &TelemetrySettings, +) -> anyhow::Result<()> { + for metric in KNOWN_METRICS { + sidecar::register_metric_ffi(sidecar_settings, telemetry_settings, metric)?; + } + Ok(()) +} + +// not implemented (difficult to count requests on the helper) +#[allow(dead_code)] +pub const RC_REQUESTS_BEFORE_RUNNING: MetricName = + MetricName("remote_config.requests_before_running"); + +pub const EVENT_RULES_LOADED: SpanMetricName = SpanMetricName("_dd.appsec.event_rules.loaded"); +pub const EVENT_RULES_FAILED: SpanMetricName = SpanMetricName("_dd.appsec.event_rules.error_count"); +pub const EVENT_RULES_ERRORS: SpanMetaName = SpanMetaName("_dd.appsec.event_rules.errors"); +pub const EVENT_RULES_VERSION: SpanMetaName = SpanMetaName("_dd.appsec.event_rules.version"); + +pub const WAF_VERSION: SpanMetaName = SpanMetaName("_dd.appsec.waf.version"); +pub const WAF_DURATION: SpanMetricName = SpanMetricName("_dd.appsec.waf.duration"); +pub const RAST_DURATION: SpanMetricName = SpanMetricName("_dd.appsec.rasp.duration"); +pub const RAST_RULE_EVALS: SpanMetricName = SpanMetricName("_dd.appsec.rasp.rule.eval"); +pub const RAST_TIMEOUTS: SpanMetricName = SpanMetricName("_dd.appsec.rasp.timeout"); + +// A "Collector" is a type of fake submitter that instead of submitting telemetry +// directly, stores it inside. It can then be converted into a generator (or implements it directly) +// for submission into the real submitter. +// See TelemetryMetricsCollector and TelemetryLogsCollector. + +#[derive(Default, Debug)] +pub struct TelemetryMetricsCollector { + metrics: HashMap>, +} +impl TelemetryMetricsCollector { + pub fn into_generator(self) -> impl TelemetryMetricsGenerator { + struct TelemetryMetricsGeneratorImpl { + metrics: Cell, + } + impl TelemetryMetricsGenerator for TelemetryMetricsGeneratorImpl { + fn generate_telemetry_metrics(&self, submitter: &mut dyn TelemetryMetricSubmitter) { + for (key, values) in self.metrics.take().metrics.into_iter() { + for (value, tags) in values { + submitter.submit_metric(key, value, tags); + } + } + } + } + TelemetryMetricsGeneratorImpl { + metrics: Cell::new(self), + } + } +} +impl TelemetryMetricSubmitter for TelemetryMetricsCollector { + fn submit_metric(&mut self, key: MetricName, value: f64, tags: TelemetryTags) { + self.metrics.entry(key).or_default().push((value, tags)); + } +} + +#[derive(Debug, Clone)] +pub struct TelemetryLog { + pub level: LogLevel, + pub identifier: String, + pub message: String, + pub stack_trace: Option, + pub tags: Option, + pub is_sensitive: bool, +} + +#[allow(dead_code)] // Used in TelemetryLogSubmitter impl +const MAX_PENDING_LOGS: usize = 100; + +pub struct TelemetryLogsCollector { + logs: Mutex>, +} +impl TelemetryLogsCollector { + pub fn new() -> Self { + Self { + logs: Mutex::new(Vec::new()), + } + } + + pub fn submit_log(&self, log: TelemetryLog) { + let mut logs = self.logs.lock().unwrap(); + + if logs.len() >= MAX_PENDING_LOGS { + log::warn!("Pending logs queue is full, dropping log"); + return; + } + + log::trace!( + "submit_log [{:?}][{}]: {}", + log.level, + log.identifier, + log.message + ); + + logs.push(log); + } +} +impl Default for TelemetryLogsCollector { + fn default() -> Self { + Self::new() + } +} +impl TelemetryLogSubmitter for TelemetryLogsCollector { + fn submit_log(&mut self, log: TelemetryLog) { + let mut logs = self.logs.lock().unwrap(); + + if logs.len() >= MAX_PENDING_LOGS { + log::warn!("Pending logs queue is full, dropping log"); + return; + } + + log::trace!( + "submit_log [{:?}][{}]: {}", + log.level, + log.identifier, + log.message + ); + + logs.push(log); + } +} +impl TelemetryLogsGenerator for TelemetryLogsCollector { + fn generate_telemetry_logs(&'_ self, submitter: &mut dyn TelemetryLogSubmitter) { + let mut logs = self.logs.lock().unwrap(); + log::debug!("Draining {} telemetry logs from collector", logs.len()); + for (i, log) in logs.drain(..).enumerate() { + log::debug!("Submitting log {} of batch", i + 1); + submitter.submit_log(log); + log::debug!("Successfully submitted log {}", i + 1); + } + log::debug!("Finished draining all logs"); + } +} diff --git a/appsec/helper-rust/src/telemetry/error_tel_ctx.rs b/appsec/helper-rust/src/telemetry/error_tel_ctx.rs new file mode 100644 index 00000000000..3773d4ac870 --- /dev/null +++ b/appsec/helper-rust/src/telemetry/error_tel_ctx.rs @@ -0,0 +1,119 @@ +use std::future::Future; +use std::sync::{Arc, RwLock}; + +use tokio::task_local; + +use crate::client::log::debug; +use crate::client::protocol::{SidecarSettings, TelemetrySettings}; + +task_local! { + static ERROR_TELEMETRY_HANDLE: ErrorTelemetryHandle; +} + +/// Run a future with an error telemetry handle set in task-local storage. +/// The handle starts empty; call `update_error_telemetry_context()` to populate it. +pub async fn with_error_telemetry_handle(fut: F) -> R +where + F: Future, +{ + ERROR_TELEMETRY_HANDLE + .scope(ErrorTelemetryHandle::new(), fut) + .await +} + +/// Update the error telemetry context for the current task. +/// Returns true if the update was successful, false if not in a scoped context. +pub fn update_error_telemetry_context( + sidecar_settings: SidecarSettings, + telemetry_settings: TelemetrySettings, +) -> bool { + let ctx = ErrorTelemetryContext { + sidecar_settings: Arc::new(sidecar_settings), + telemetry_settings: Arc::new(telemetry_settings), + }; + ERROR_TELEMETRY_HANDLE + .try_with(|handle| handle.set(ctx)) + .is_ok() +} + +/// Clear the error telemetry context for the current task. +/// Returns true if the clear was successful, false if not in a scoped context. +pub fn clear_error_telemetry_context() -> bool { + ERROR_TELEMETRY_HANDLE + .try_with(|handle| handle.clear()) + .is_ok() +} + +pub fn get_context_log_submitter() -> impl super::TelemetryLogSubmitter { + struct ContextTelemetryLogSubmitter {} + impl super::TelemetryLogSubmitter for ContextTelemetryLogSubmitter { + fn submit_log(&mut self, log: super::TelemetryLog) { + let Some(ctx) = get_error_telemetry_context() else { + debug!( + "Cannot submit telemetry log {:?}: no error telemetry context", + log + ); + return; + }; + + super::TelemetrySidecarLogSubmitter::create( + &ctx.sidecar_settings, + &ctx.telemetry_settings, + ) + .submit_log(log); + } + } + + ContextTelemetryLogSubmitter {} +} + +/// Context for error telemetry submission. +/// Both settings must be present for telemetry to be submitted. +#[derive(Clone)] +pub struct ErrorTelemetryContext { + pub sidecar_settings: Arc, + pub telemetry_settings: Arc, +} + +/// Handle to the task-local error telemetry context. +/// This handle is set once at task start via `.scope()`, but the inner contents +/// can be updated at any time (e.g., when settings change via config_sync). +#[derive(Clone)] +pub struct ErrorTelemetryHandle(Arc>>); + +impl ErrorTelemetryHandle { + pub fn new() -> Self { + Self(Arc::new(RwLock::new(None))) + } + + pub fn set(&self, ctx: ErrorTelemetryContext) { + if let Ok(mut guard) = self.0.write() { + *guard = Some(ctx); + } + } + + pub fn clear(&self) { + if let Ok(mut guard) = self.0.write() { + *guard = None; + } + } + + pub fn get(&self) -> Option { + self.0.read().ok().and_then(|guard| guard.clone()) + } +} + +impl Default for ErrorTelemetryHandle { + fn default() -> Self { + Self::new() + } +} + +/// Get the current error telemetry context for this task, if available. +/// Returns None if called outside of a scoped context or if no context has been set. +fn get_error_telemetry_context() -> Option { + ERROR_TELEMETRY_HANDLE + .try_with(|handle| handle.get()) + .ok() + .flatten() +} diff --git a/appsec/helper-rust/src/telemetry/sidecar.rs b/appsec/helper-rust/src/telemetry/sidecar.rs new file mode 100644 index 00000000000..17905abd86b --- /dev/null +++ b/appsec/helper-rust/src/telemetry/sidecar.rs @@ -0,0 +1,466 @@ +use std::cell::Cell; +use std::ffi::c_char; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; +use std::task::Poll; +use std::time::{Duration, Instant}; + +use futures::future::Shared; +use futures::task::Context; + +use crate::client::log::{debug, info, warning}; +use crate::client::protocol::{SidecarSettings, TelemetrySettings}; +use crate::ffi::sidecar_ffi::{ + ddog_CharSlice, ddog_Error, ddog_Error_drop, ddog_Error_message, ddog_LogLevel, + ddog_LogLevel_DDOG_LOG_LEVEL_DEBUG, ddog_LogLevel_DDOG_LOG_LEVEL_ERROR, + ddog_LogLevel_DDOG_LOG_LEVEL_WARN, ddog_MaybeError, ddog_MetricNamespace, + ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_APPSEC, ddog_MetricType, + ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR, ddog_SidecarTransport, + ddog_sidecar_connect, ddog_sidecar_enqueue_telemetry_log, + ddog_sidecar_enqueue_telemetry_metric, ddog_sidecar_enqueue_telemetry_point, ddog_sidecar_ping, + ddog_sidecar_transport_drop, +}; +use crate::ffi::SidecarSymbol; +use crate::sidecar_symbol; +use crate::telemetry::{ + KnownMetric, TelemetryLogSubmitter, TelemetryMetricSubmitter, TelemetryTags, +}; + +use super::{LogLevel, MetricName, TelemetryLog}; + +type DdogSidecarEnqueueTelemetryLogFn = unsafe extern "C" fn( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + identifier_ffi: ddog_CharSlice, + level: ddog_LogLevel, + message_ffi: ddog_CharSlice, + stack_trace_ffi: *mut ddog_CharSlice, + tags_ffi: *mut ddog_CharSlice, + is_sensitive: bool, +) -> ddog_MaybeError; +type DdogSidecarEnqueueTelemetryPointFn = unsafe extern "C" fn( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + metric_name_ffi: ddog_CharSlice, + value: f64, + tags_ffi: *mut ddog_CharSlice, +) -> ddog_MaybeError; +type DdogSidecarEnqueueTelemetryMetricFn = unsafe extern "C" fn( + session_id_ffi: ddog_CharSlice, + runtime_id_ffi: ddog_CharSlice, + service_name_ffi: ddog_CharSlice, + env_name_ffi: ddog_CharSlice, + metric_name_ffi: ddog_CharSlice, + metric_type: ddog_MetricType, + metric_namespace: ddog_MetricNamespace, +) -> ddog_MaybeError; +type DdogErrorDropFn = unsafe extern "C" fn(*mut ddog_Error); +type DdogErrorMessageFn = unsafe extern "C" fn(*const ddog_Error) -> ddog_CharSlice; +type DdogSidecarConnectFn = + unsafe extern "C" fn(*mut *mut ddog_SidecarTransport) -> ddog_MaybeError; +type DdogSidecarPingFn = unsafe extern "C" fn(*mut *mut ddog_SidecarTransport) -> ddog_MaybeError; +type DdogSidecarTransportDropFn = unsafe extern "C" fn(*mut ddog_SidecarTransport); + +static RESOLUTION_STATUS: AtomicBool = AtomicBool::new(false); + +#[repr(u8)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum SidecarStatus { + Unknown = 0, + Ready = 1, + Failed = 2, +} + +static SIDECAR_STATUS: AtomicU8 = AtomicU8::new(SidecarStatus::Unknown as u8); + +sidecar_symbol!( + static ENQUEUE_TELEMETRY_LOG = DdogSidecarEnqueueTelemetryLogFn : ddog_sidecar_enqueue_telemetry_log +); +sidecar_symbol!( + static ENQUEUE_TELEMETRY_POINT = DdogSidecarEnqueueTelemetryPointFn : ddog_sidecar_enqueue_telemetry_point +); +sidecar_symbol!( + static ENQUEUE_TELEMETRY_METRIC = DdogSidecarEnqueueTelemetryMetricFn : ddog_sidecar_enqueue_telemetry_metric +); +sidecar_symbol!( + static ERROR_DROP = DdogErrorDropFn : ddog_Error_drop +); +sidecar_symbol!( + static ERROR_MESSAGE = DdogErrorMessageFn : ddog_Error_message +); +sidecar_symbol!( + static SIDECAR_CONNECT = DdogSidecarConnectFn : ddog_sidecar_connect +); +sidecar_symbol!( + static SIDECAR_PING = DdogSidecarPingFn : ddog_sidecar_ping +); +sidecar_symbol!( + static SIDECAR_TRANSPORT_DROP = DdogSidecarTransportDropFn : ddog_sidecar_transport_drop +); + +pub struct TelemetrySidecarLogSubmitter<'a> { + session_id: &'a str, + runtime_id: &'a str, + service_name: &'a str, + env_name: &'a str, +} + +impl<'a> TelemetrySidecarLogSubmitter<'a> { + pub fn create( + sidecar_settings: &'a SidecarSettings, + telemetry_settings: &'a TelemetrySettings, + ) -> Box { + struct NoopTelemetryLogSubmitter; + impl TelemetryLogSubmitter for NoopTelemetryLogSubmitter { + fn submit_log(&mut self, log: TelemetryLog) { + debug!( + "Not submitting telemetry log: sidecar symbols not resolved, log={:?}", + log + ); + } + } + + if !RESOLUTION_STATUS.load(Ordering::Acquire) { + info!("Sidecar symbols for telemetry not resolved, skipping log submission"); + return Box::new(NoopTelemetryLogSubmitter); + } + if SIDECAR_STATUS.load(Ordering::Relaxed) != SidecarStatus::Ready as u8 { + info!("Sidecar is not ready, skipping log submission"); + return Box::new(NoopTelemetryLogSubmitter); + } + + Box::new(Self { + session_id: &sidecar_settings.session_id, + runtime_id: &sidecar_settings.runtime_id, + service_name: &telemetry_settings.service_name, + env_name: &telemetry_settings.env_name, + }) + } +} + +fn to_ddog_log_level(level: LogLevel) -> ddog_LogLevel { + match level { + LogLevel::Error => ddog_LogLevel_DDOG_LOG_LEVEL_ERROR, + LogLevel::Warn => ddog_LogLevel_DDOG_LOG_LEVEL_WARN, + LogLevel::Debug => ddog_LogLevel_DDOG_LOG_LEVEL_DEBUG, + } +} + +fn char_slice_from_str(s: &str) -> ddog_CharSlice { + ddog_CharSlice { + ptr: s.as_ptr() as *const c_char, + len: s.len(), + } +} + +struct MaybeErrorRAII { + maybe_error: ddog_MaybeError, +} +impl Drop for MaybeErrorRAII { + fn drop(&mut self) { + unsafe { + if self.maybe_error.tag == ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR { + ERROR_DROP(&mut self.maybe_error.__bindgen_anon_1.__bindgen_anon_1.some); + } + } + } +} +impl From for Option { + fn from(value: MaybeErrorRAII) -> Self { + if value.maybe_error.tag == ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR { + let msg = + unsafe { ERROR_MESSAGE(&value.maybe_error.__bindgen_anon_1.__bindgen_anon_1.some) }; + if msg.ptr.is_null() || msg.len == 0 { + return Some(String::new()); + } + + let msg = unsafe { std::slice::from_raw_parts(msg.ptr as *const u8, msg.len) }; + Some(String::from_utf8_lossy(msg).into_owned()) + } else { + None + } + } +} +impl From for MaybeErrorRAII { + fn from(maybe_error: ddog_MaybeError) -> Self { + Self { maybe_error } + } +} + +impl TelemetryLogSubmitter for TelemetrySidecarLogSubmitter<'_> { + fn submit_log(&mut self, mut log: TelemetryLog) { + let mut tags = log.tags.take().unwrap_or_default(); + tags.add("helper_runtime", "rust"); + log.tags = Some(tags); + + info!( + "Submitting telemetry log to sidecar: identifier={}, level={:?} (raw={}), message={}", + log.identifier, log.level, log.level as u8, log.message + ); + + let session_id = char_slice_from_str(self.session_id); + let runtime_id = char_slice_from_str(self.runtime_id); + let service_name = char_slice_from_str(self.service_name); + let env_name = char_slice_from_str(self.env_name); + let identifier = char_slice_from_str(&log.identifier); + let message = char_slice_from_str(&log.message); + + let tags_string = log.tags.map(|t| t.into_string()); + let tags_slice = tags_string.as_ref().map(|t| char_slice_from_str(t)); + + let stack_trace_slice = log.stack_trace.as_ref().map(|st| char_slice_from_str(st)); + + let result: ddog_MaybeError = unsafe { + ENQUEUE_TELEMETRY_LOG( + session_id, + runtime_id, + service_name, + env_name, + identifier, + to_ddog_log_level(log.level), + message, + stack_trace_slice + .as_ref() + .map_or(std::ptr::null_mut(), |s| s as *const _ as *mut _), + tags_slice + .as_ref() + .map_or(std::ptr::null_mut(), |t| t as *const _ as *mut _), + log.is_sensitive, + ) + }; + let result: MaybeErrorRAII = result.into(); + + if let Some(error_msg) = Into::>::into(result) { + info!("Failed to enqueue telemetry log, error: {}", error_msg); + } + } +} + +pub fn resolve_symbols() -> anyhow::Result<()> { + ENQUEUE_TELEMETRY_LOG.resolve()?; + ENQUEUE_TELEMETRY_POINT.resolve()?; + ENQUEUE_TELEMETRY_METRIC.resolve()?; + ERROR_DROP.resolve()?; + ERROR_MESSAGE.resolve()?; + SIDECAR_CONNECT.resolve()?; + SIDECAR_PING.resolve()?; + SIDECAR_TRANSPORT_DROP.resolve()?; + RESOLUTION_STATUS.store(true, Ordering::Release); + Ok(()) +} + +pub struct SidecarReadyFuture { + attempt: u32, + sleep: Option>>, +} + +impl SidecarReadyFuture { + const MAX_ATTEMPTS: u32 = 50; + const SLEEP_DURATION: Duration = Duration::from_millis(100); + + pub fn create() -> Shared { + let future = Self { + attempt: 0, + sleep: None, + }; + futures::FutureExt::shared(future) + } + + fn try_connect_and_ping(&self) -> bool { + let mut transport: *mut ddog_SidecarTransport = std::ptr::null_mut(); + + let mut connect_result = unsafe { SIDECAR_CONNECT(&mut transport as *mut _) }; + + if connect_result.tag == ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR { + unsafe { ERROR_DROP(&mut connect_result.__bindgen_anon_1.__bindgen_anon_1.some) }; + return false; + } + + let ping_result = unsafe { SIDECAR_PING(&mut transport as *mut _) }; + unsafe { SIDECAR_TRANSPORT_DROP(transport) }; + + ping_result.tag != ddog_Option_Error_Tag_DDOG_OPTION_ERROR_SOME_ERROR + } +} + +impl Future for SidecarReadyFuture { + type Output = SidecarStatus; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + loop { + if let Some(ref mut sleep) = self.sleep { + match sleep.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(()) => { + self.sleep = None; + } + } + } + + if self.attempt >= Self::MAX_ATTEMPTS { + warning!( + "Sidecar did not become ready after {} attempts, not trying again", + Self::MAX_ATTEMPTS + ); + SIDECAR_STATUS.store(SidecarStatus::Failed as u8, Ordering::Release); + return Poll::Ready(SidecarStatus::Failed); + } + + self.attempt += 1; + + if self.try_connect_and_ping() { + info!("Sidecar is ready after {} attempts", self.attempt); + SIDECAR_STATUS.store(SidecarStatus::Ready as u8, Ordering::Release); + return Poll::Ready(SidecarStatus::Ready); + } + + debug!( + "Sidecar not ready yet (attempt {}), waiting...", + self.attempt + ); + self.sleep = Some(Box::pin(tokio::time::sleep(Self::SLEEP_DURATION))); + } + } +} + +pub(super) fn register_metric_ffi( + sidecar_settings: &SidecarSettings, + telemetry_settings: &TelemetrySettings, + metric: &KnownMetric, +) -> anyhow::Result<()> { + if !RESOLUTION_STATUS.load(Ordering::Acquire) { + anyhow::bail!("Sidecar symbols not resolved, skipping metric registration") + } + + if SIDECAR_STATUS.load(Ordering::Relaxed) != SidecarStatus::Ready as u8 { + anyhow::bail!("Sidecar is not ready, skipping metric registration"); + } + + let session_id = char_slice_from_str(&sidecar_settings.session_id); + let runtime_id = char_slice_from_str(&sidecar_settings.runtime_id); + let service_name = char_slice_from_str(&telemetry_settings.service_name); + let env_name = char_slice_from_str(&telemetry_settings.env_name); + let metric_name_slice = char_slice_from_str(metric.name.0); + + let result: ddog_MaybeError = unsafe { + ENQUEUE_TELEMETRY_METRIC( + session_id, + runtime_id, + service_name, + env_name, + metric_name_slice, + metric.metric_type, + ddog_MetricNamespace_DDOG_METRIC_NAMESPACE_APPSEC, + ) + }; + let result: MaybeErrorRAII = result.into(); + + if let Some(error_msg) = Into::>::into(result) { + anyhow::bail!( + "Failed to register metric {}, error: {}", + metric.name.0, + error_msg + ); + } + Ok(()) +} + +pub struct TelemetrySidecarMetricSubmitter<'a> { + session_id: &'a str, + runtime_id: &'a str, + service_name: &'a str, + env_name: &'a str, +} + +impl<'a> TelemetrySidecarMetricSubmitter<'a> { + pub fn create<'b>( + sidecar_settings: &'a SidecarSettings, + telemetry_settings: &'a TelemetrySettings, + last_registration_time: &'b Cell>, + ) -> Box { + if !RESOLUTION_STATUS.load(Ordering::Acquire) { + warning!("Sidecar symbols for telemetry not resolved, skipping metric submission"); + return Self::noop(); + } + + // Telemetry client is deleted after 30 mins with no activity. So we may need + // to refresh at least every 30 mins + const METRICS_REGISTRATION_REFRESH: Duration = Duration::from_secs(25 * 60); + + let needs_registration = last_registration_time + .get() + .is_none_or(|i| i.elapsed() < METRICS_REGISTRATION_REFRESH); + if needs_registration { + if let Err(err) = super::register_known_metrics(sidecar_settings, telemetry_settings) { + warning!("Failed to register known metrics: {}", err); + return Self::noop(); + } + last_registration_time.set(Some(Instant::now())); + } + + Box::new(Self { + session_id: &sidecar_settings.session_id, + runtime_id: &sidecar_settings.runtime_id, + service_name: &telemetry_settings.service_name, + env_name: &telemetry_settings.env_name, + }) + } + + pub fn noop() -> Box { + struct NoopTelemetryMetricSubmitter; + impl TelemetryMetricSubmitter for NoopTelemetryMetricSubmitter { + fn submit_metric(&mut self, key: MetricName, _value: f64, _tags: TelemetryTags) { + debug!( + "Not submitting telemetry metric: key={} (see earlier warning)", + key.0 + ); + } + } + + Box::new(NoopTelemetryMetricSubmitter) + } +} + +impl TelemetryMetricSubmitter for TelemetrySidecarMetricSubmitter<'_> { + fn submit_metric(&mut self, key: MetricName, value: f64, mut tags: TelemetryTags) { + tags.add("helper_runtime", "rust"); + + info!( + "Submitting telemetry metric to sidecar: metric={}, value={}, tags={}", + key.0, + value, + tags.clone().into_string() + ); + + let session_id = char_slice_from_str(self.session_id); + let runtime_id = char_slice_from_str(self.runtime_id); + let service_name = char_slice_from_str(self.service_name); + let env_name = char_slice_from_str(self.env_name); + let metric_name = char_slice_from_str(key.0); + + let tags_string = tags.into_string(); + let tags_slice = char_slice_from_str(&tags_string); + + let result: ddog_MaybeError = unsafe { + ENQUEUE_TELEMETRY_POINT( + session_id, + runtime_id, + service_name, + env_name, + metric_name, + value, + &tags_slice as *const _ as *mut _, + ) + }; + let result: MaybeErrorRAII = result.into(); + + if let Some(error_msg) = Into::>::into(result) { + info!("Failed to enqueue telemetry metric, error: {}", error_msg); + } + } +} diff --git a/appsec/helper-rust/src/telemetry/tel_aware_logger.rs b/appsec/helper-rust/src/telemetry/tel_aware_logger.rs new file mode 100644 index 00000000000..866c1303dbc --- /dev/null +++ b/appsec/helper-rust/src/telemetry/tel_aware_logger.rs @@ -0,0 +1,326 @@ +use log::kv::{Key, VisitSource}; +use log::{Level, Log, Metadata, Record}; +use std::backtrace::Backtrace; +use std::borrow::Cow; +use std::cell::Cell; + +use super::{LogLevel, TelemetryLog, TelemetryTags}; +use crate::client::log::ANYHOW_BACKTRACE_KEY; +use crate::telemetry::error_tel_ctx::get_context_log_submitter; +use crate::telemetry::TelemetryLogSubmitter; + +/// A composite logger that dispatches to the primary logger +/// and submits error-level logs to telemetry. +pub struct TelemetryAwareLogger { + delegate: Box, +} + +impl TelemetryAwareLogger { + pub fn new(delegate: Box) -> Self { + Self { delegate } + } +} + +impl Log for TelemetryAwareLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + self.delegate.enabled(metadata) + } + + fn log(&self, record: &Record) { + self.delegate.log(record); + + if record.level() != Level::Error { + return; + } + + submit_error_to_telemetry(record); + } + + fn flush(&self) { + self.delegate.flush(); + } +} + +// Recursion guard: prevents infinite loops if telemetry submission logs an error +thread_local! { + static IN_ERROR_HANDLER: Cell = const { Cell::new(false) }; +} + +struct RecursionGuard; +impl RecursionGuard { + fn enter() -> Option { + IN_ERROR_HANDLER.with(|cell| { + if cell.get() { + None + } else { + cell.set(true); + Some(RecursionGuard) + } + }) + } +} +impl Drop for RecursionGuard { + fn drop(&mut self) { + IN_ERROR_HANDLER.with(|cell| cell.set(false)); + } +} + +// Rate limiter: allows at most one error per interval per thread +thread_local! { + static LAST_SUBMIT_TIME: Cell = const { Cell::new(0) }; +} + +const MIN_INTERVAL_MS: u64 = 1000; + +fn should_rate_limit() -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + + LAST_SUBMIT_TIME.with(|cell| { + let last = cell.get(); + if now.saturating_sub(last) < MIN_INTERVAL_MS { + true + } else { + cell.set(now); + false + } + }) +} + +fn submit_error_to_telemetry(record: &Record) { + let Some(_guard) = RecursionGuard::enter() else { + return; + }; + + let mut tags = TelemetryTags::new(); + tags.add("log_type", "helper::logged_error"); + if let Some(module) = record.module_path() { + tags.add("module", module); + } + + let message = format!("{}", record.args()); + + let location = if let (Some(module), Some(line)) = (record.module_path(), record.line()) { + Cow::Owned(format!("{}:{}", module, line)) + } else if let Some(module) = record.module_path() { + Cow::Borrowed(module) + } else { + Cow::Borrowed("unknown") + }; + + if should_rate_limit() { + return; + } + + let stack_trace = extract_anyhow_backtrace(record).or_else(|| { + // Fall back to capturing backtrace at the logger (less useful but better than nothing) + let backtrace = Backtrace::force_capture(); + match backtrace.status() { + std::backtrace::BacktraceStatus::Captured => Some(backtrace.to_string()), + _ => None, + } + }); + + let log = TelemetryLog { + level: LogLevel::Error, + identifier: format!("helper::{}", location), + message: format!("{} at {}", message, location), + stack_trace, + tags: Some(tags), + is_sensitive: false, + }; + + let mut submitter = get_context_log_submitter(); + submitter.submit_log(log); +} + +/// Visitor to extract anyhow backtrace from log record's key-values +struct BacktraceExtractor { + backtrace: Option, +} + +impl<'kvs> VisitSource<'kvs> for BacktraceExtractor { + fn visit_pair( + &mut self, + key: Key<'kvs>, + value: log::kv::Value<'kvs>, + ) -> Result<(), log::kv::Error> { + if key.as_str() == ANYHOW_BACKTRACE_KEY { + self.backtrace = Some(value.to_string()); + } + Ok(()) + } +} + +fn extract_anyhow_backtrace(record: &Record) -> Option { + let mut extractor = BacktraceExtractor { backtrace: None }; + let _ = record.key_values().visit(&mut extractor); + extractor.backtrace +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + + struct TestLogger { + logs: Arc>>, + } + + impl Log for TestLogger { + fn enabled(&self, _metadata: &Metadata) -> bool { + true + } + + fn log(&self, record: &Record) { + self.logs + .lock() + .unwrap() + .push(format!("[{}] {}", record.level(), record.args())); + } + + fn flush(&self) {} + } + + #[test] + fn test_recursion_guard() { + assert!(RecursionGuard::enter().is_some()); + { + let _guard1 = RecursionGuard::enter().unwrap(); + assert!(RecursionGuard::enter().is_none()); + assert!(RecursionGuard::enter().is_none()); + } + assert!(RecursionGuard::enter().is_some()); + } + + #[test] + fn test_rate_limiting() { + LAST_SUBMIT_TIME.with(|cell| cell.set(0)); + + assert!(!should_rate_limit()); + + assert!(should_rate_limit()); + assert!(should_rate_limit()); + + LAST_SUBMIT_TIME.with(|cell| { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + cell.set(now - MIN_INTERVAL_MS - 1); + }); + + assert!(!should_rate_limit()); + } + + #[test] + fn test_composite_logger_delegates_to_primary() { + let logs = Arc::new(Mutex::new(Vec::new())); + let primary = Box::new(TestLogger { logs: logs.clone() }); + let composite = TelemetryAwareLogger::new(primary); + + let record = log::Record::builder() + .args(format_args!("test message")) + .level(Level::Info) + .build(); + + composite.log(&record); + + let captured = logs.lock().unwrap(); + assert_eq!(captured.len(), 1); + assert_eq!(captured[0], "[INFO] test message"); + } + + #[test] + fn test_composite_logger_handles_all_levels() { + let logs = Arc::new(Mutex::new(Vec::new())); + let primary = Box::new(TestLogger { logs: logs.clone() }); + let composite = TelemetryAwareLogger::new(primary); + + macro_rules! log_level { + ($level:expr, $msg:literal) => {{ + let record = log::Record::builder() + .args(format_args!($msg)) + .level($level) + .build(); + composite.log(&record); + }}; + } + + log_level!(Level::Trace, "trace message"); + log_level!(Level::Debug, "debug message"); + log_level!(Level::Info, "info message"); + log_level!(Level::Warn, "warn message"); + log_level!(Level::Error, "error message"); + + let captured = logs.lock().unwrap(); + assert_eq!(captured.len(), 5); + } + + #[test] + fn test_extract_anyhow_backtrace_with_key() { + use log::kv::{self, ToValue}; + + struct TestKvs<'a> { + backtrace: &'a str, + } + + impl<'kvs> kv::Source for TestKvs<'kvs> { + fn visit<'a>(&'a self, visitor: &mut dyn kv::VisitSource<'a>) -> Result<(), kv::Error> { + visitor.visit_pair( + kv::Key::from_str(ANYHOW_BACKTRACE_KEY), + self.backtrace.to_value(), + ) + } + } + + let kvs = TestKvs { + backtrace: "test backtrace content", + }; + let record = log::Record::builder() + .args(format_args!("test")) + .level(Level::Error) + .key_values(&kvs) + .build(); + + let extracted = extract_anyhow_backtrace(&record); + assert_eq!(extracted, Some("test backtrace content".to_string())); + } + + #[test] + fn test_extract_anyhow_backtrace_without_key() { + let record = log::Record::builder() + .args(format_args!("test")) + .level(Level::Error) + .build(); + + let extracted = extract_anyhow_backtrace(&record); + assert!(extracted.is_none()); + } + + #[test] + fn test_extract_anyhow_backtrace_with_other_keys() { + use log::kv::{self, ToValue}; + + struct TestKvs; + + impl kv::Source for TestKvs { + fn visit<'a>(&'a self, visitor: &mut dyn kv::VisitSource<'a>) -> Result<(), kv::Error> { + visitor.visit_pair(kv::Key::from_str("other_key"), "other_value".to_value())?; + visitor.visit_pair(kv::Key::from_str("another_key"), 42i32.to_value()) + } + } + + let kvs = TestKvs; + let record = log::Record::builder() + .args(format_args!("test")) + .level(Level::Error) + .key_values(&kvs) + .build(); + + let extracted = extract_anyhow_backtrace(&record); + assert!(extracted.is_none()); + } +} diff --git a/appsec/helper-rust/test_sidecar_lib.c b/appsec/helper-rust/test_sidecar_lib.c new file mode 100644 index 00000000000..99883017166 --- /dev/null +++ b/appsec/helper-rust/test_sidecar_lib.c @@ -0,0 +1,3 @@ +int test_add(int a, int b) { + return a + b; +} diff --git a/appsec/src/extension/commands/client_init.c b/appsec/src/extension/commands/client_init.c index 7ef8be6fc52..161c4161767 100644 --- a/appsec/src/extension/commands/client_init.c +++ b/appsec/src/extension/commands/client_init.c @@ -11,14 +11,17 @@ #include "../configuration.h" #include "../ddappsec.h" #include "../ddtrace.h" +#include "../helper_process.h" #include "../logging.h" #include "../msgpack_helpers.h" #include "../php_compat.h" +#include "../string_helpers.h" #include "../version.h" #include "client_init.h" static dd_result _pack_command(mpack_writer_t *nonnull w, void *nullable ctx); static dd_result _process_response(mpack_node_t root, void *nullable ctx); +static void _process_helper_runtime(mpack_node_t root); static void _process_meta_and_metrics( mpack_node_t root, struct req_info *nonnull ctx); @@ -150,7 +153,7 @@ static dd_result _check_helper_version(mpack_node_t root); static dd_result _process_response( mpack_node_t root, ATTR_UNUSED void *nullable ctx) { - // Add any tags and metrics provided by the helper + _process_helper_runtime(root); _process_meta_and_metrics(root, ctx); // check verdict @@ -191,9 +194,27 @@ static dd_result _process_response( return dd_error; } +static void _process_helper_runtime(mpack_node_t root) +{ +#define HELPER_RUNTIME_INDEX 5 + mpack_node_t runtime_node = mpack_node_array_at(root, HELPER_RUNTIME_INDEX); + if (mpack_node_type(runtime_node) == mpack_type_str) { + const char *runtime = mpack_node_str(runtime_node); + size_t runtime_len = mpack_node_strlen(runtime_node); + if (STR_CONS_EQ(runtime, runtime_len, "rust")) { + dd_helper_set_runtime(HELPER_RUNTIME_RUST); + } else if (STR_CONS_EQ(runtime, runtime_len, "cpp")) { + dd_helper_set_runtime(HELPER_RUNTIME_CPP); + } else { + dd_helper_set_runtime(HELPER_RUNTIME_UNKNOWN); + } + } +} + static void _process_meta_and_metrics( mpack_node_t root, struct req_info *nonnull ctx) { + mpack_node_t meta = mpack_node_array_at(root, 3); zend_object *span = ctx->root_span; if (!span) { mlog( @@ -201,7 +222,6 @@ static void _process_meta_and_metrics( return; } - mpack_node_t meta = mpack_node_array_at(root, 3); if (mpack_node_map_count(meta) > 0) { dd_command_process_meta(meta, span); } diff --git a/appsec/src/extension/commands/request_exec.c b/appsec/src/extension/commands/request_exec.c index 20e1f701598..e3ae23ed199 100644 --- a/appsec/src/extension/commands/request_exec.c +++ b/appsec/src/extension/commands/request_exec.c @@ -9,6 +9,7 @@ #include "../ddappsec.h" #include "../logging.h" #include "../msgpack_helpers.h" +#include #include #include #include @@ -16,7 +17,9 @@ struct ctx { struct req_info req_info; // dd_command_proc_resp_verd_span_data expect it zend_string *nullable rasp_rule; - zval *nonnull data; + zend_string *nullable subctx_id; + bool subctx_last_call; + zend_array *nonnull data; }; static dd_result _pack_command(mpack_writer_t *nonnull w, void *nonnull ctx); @@ -30,16 +33,14 @@ static const dd_command_spec _spec = { .config_features_cb = dd_command_process_config_features_unexpected, }; -dd_result dd_request_exec(dd_conn *nonnull conn, zval *nonnull data, - zend_string *nullable rasp_rule, struct block_params *nonnull block_params) +dd_result dd_request_exec(dd_conn *nonnull conn, zend_array *nonnull data, + const struct req_exec_opts *nonnull opts, + struct block_params *nonnull block_params) { - if (Z_TYPE_P(data) != IS_ARRAY) { - mlog(dd_log_debug, "Invalid data provided to command request_exec, " - "expected hash table."); - return dd_error; - } - - struct ctx ctx = {.rasp_rule = rasp_rule, .data = data}; + struct ctx ctx = {.data = data, + .rasp_rule = opts->rasp_rule, + .subctx_id = opts->subctx_id, + .subctx_last_call = opts->subctx_last_call}; dd_result res = dd_command_exec_req_info(conn, &_spec, &ctx.req_info); @@ -53,14 +54,31 @@ static dd_result _pack_command(mpack_writer_t *nonnull w, void *nonnull _ctx) assert(_ctx != NULL); struct ctx *ctx = _ctx; - dd_mpack_write_nullable_zstr(w, ctx->rasp_rule); - dd_mpack_limits limits = dd_mpack_def_limits; - dd_mpack_write_zval_lim(w, ctx->data, &limits); + dd_mpack_write_array_lim(w, ctx->data, &limits); + + size_t num_map_elems = + (ctx->rasp_rule != NULL) + (ctx->subctx_id != NULL) * 2; + mpack_start_map(w, num_map_elems); if (dd_mpack_limits_reached(&limits)) { mlog(dd_log_info, "Limits reched when serializing request exec data"); } + if (ctx->rasp_rule != NULL) { + dd_mpack_write_lstr(w, "rasp_rule"); + dd_mpack_write_zstr(w, ctx->rasp_rule); + } + + if (ctx->subctx_id != NULL) { + dd_mpack_write_lstr(w, "subctx_id"); + dd_mpack_write_nullable_zstr(w, ctx->subctx_id); + + dd_mpack_write_lstr(w, "subctx_last_call"); + mpack_write_bool(w, ctx->subctx_last_call); + } + + mpack_finish_map(w); + return dd_success; } diff --git a/appsec/src/extension/commands/request_exec.h b/appsec/src/extension/commands/request_exec.h index 70bf538b1d6..46bb67767d2 100644 --- a/appsec/src/extension/commands/request_exec.h +++ b/appsec/src/extension/commands/request_exec.h @@ -11,5 +11,12 @@ #include "../network.h" #include "../request_abort.h" -dd_result dd_request_exec(dd_conn *nonnull conn, zval *nonnull data, - zend_string *nullable rasp_rule, struct block_params *nonnull block_params); +struct req_exec_opts { + zend_string *nullable rasp_rule; + zend_string *nullable subctx_id; + bool subctx_last_call; +}; + +dd_result dd_request_exec(dd_conn *nonnull conn, zend_array *nonnull data, + const struct req_exec_opts *nonnull opts, + struct block_params *nonnull block_params); diff --git a/appsec/src/extension/configuration.h b/appsec/src/extension/configuration.h index 0794b8761d3..a1137ef0f22 100644 --- a/appsec/src/extension/configuration.h +++ b/appsec/src/extension/configuration.h @@ -28,6 +28,12 @@ extern bool runtime_config_first_init; #define DD_BASE(path) "/opt/datadog-php/" +#if PHP_VERSION_ID >= 80500 +#define DD_APPSEC_HELPER_RUST_REDIRECTION_DEFAULT "true" +#else +#define DD_APPSEC_HELPER_RUST_REDIRECTION_DEFAULT "false" +#endif + // clang-format off #define DD_CONFIGURATION_GENERAL \ CONFIG(BOOL, DD_APPSEC_ENABLED, "false", .ini_change = zai_config_system_ini_change) \ @@ -42,10 +48,12 @@ extern bool runtime_config_first_init; SYSCFG(BOOL, DD_APPSEC_TESTING_ABORT_RINIT, "false") \ SYSCFG(BOOL, DD_APPSEC_TESTING_RAW_BODY, "false") \ SYSCFG(BOOL, DD_APPSEC_TESTING_HELPER_METRICS, "false") \ + SYSCFG(BOOL, DD_APPSEC_TESTING_INVALID_COMMAND, "false") \ CONFIG(CUSTOM(INT), DD_APPSEC_LOG_LEVEL, "warn", .parser = dd_parse_log_level) \ SYSCFG(STRING, DD_APPSEC_LOG_FILE, "php_error_reporting") \ SYSCFG(BOOL, DD_APPSEC_HELPER_LAUNCH, "true") \ CONFIG(STRING, DD_APPSEC_HELPER_PATH, DD_BASE("bin/libddappsec-helper.so")) \ + SYSCFG(BOOL, DD_APPSEC_HELPER_RUST_REDIRECTION, DD_APPSEC_HELPER_RUST_REDIRECTION_DEFAULT) \ SYSCFG(BOOL, DD_APPSEC_STACK_TRACE_ENABLED, "true") \ SYSCFG(BOOL, DD_APPSEC_RASP_ENABLED , "true") \ SYSCFG(INT, DD_APPSEC_MAX_STACK_TRACE_DEPTH, "32") \ diff --git a/appsec/src/extension/ddappsec.c b/appsec/src/extension/ddappsec.c index 5c06e8f62c6..ff4d71ace4c 100644 --- a/appsec/src/extension/ddappsec.c +++ b/appsec/src/extension/ddappsec.c @@ -22,6 +22,7 @@ #include "commands/client_init.h" #include "commands/request_exec.h" #include "commands_ctx.h" +#include "commands_helpers.h" #include "compatibility.h" #include "configuration.h" #include "ddappsec.h" @@ -463,8 +464,8 @@ static PHP_FUNCTION(datadog_appsec_testing_stop_for_debugger) static PHP_FUNCTION(datadog_appsec_testing_request_exec) { - zval *data = NULL; - if (zend_parse_parameters(ZEND_NUM_ARGS(), "z", &data) != SUCCESS) { + zend_array *data = NULL; + if (zend_parse_parameters(ZEND_NUM_ARGS(), "h", &data) != SUCCESS) { RETURN_FALSE; } @@ -479,7 +480,8 @@ static PHP_FUNCTION(datadog_appsec_testing_request_exec) } struct block_params block_params = {0}; - if (dd_request_exec(conn, data, false, &block_params) != dd_success) { + struct req_exec_opts opts = {0}; + if (dd_request_exec(conn, data, &opts, &block_params) != dd_success) { RETVAL_FALSE; } else { RETVAL_TRUE; @@ -487,10 +489,67 @@ static PHP_FUNCTION(datadog_appsec_testing_request_exec) dd_request_abort_destroy_block_params(&block_params); } +static dd_result _pack_invalid_command( + mpack_writer_t *nonnull w, ATTR_UNUSED void *nullable ctx) +{ + UNUSED(ctx); + mpack_start_map(w, 1); + dd_mpack_write_lstr(w, "foo"); + dd_mpack_write_lstr(w, "bar"); + mpack_finish_map(w); + return dd_success; +} + +static dd_result _process_invalid_response( + mpack_node_t root, ATTR_UNUSED void *nullable ctx) +{ + UNUSED(root); + UNUSED(ctx); + return dd_success; +} + +static PHP_FUNCTION(datadog_appsec_testing_send_invalid_command) +{ + if (zend_parse_parameters_none() == FAILURE) { + RETURN_FALSE; + } + + dd_conn *conn = dd_helper_mgr_cur_conn(); + if (conn == NULL) { + struct req_info ctx = { + .root_span = dd_trace_get_active_root_span(), + }; + conn = + dd_helper_mgr_acquire_conn((client_init_func)dd_client_init, &ctx); + if (conn == NULL) { + mlog_g(dd_log_debug, + "Failed to acquire connection for invalid message test"); + RETURN_FALSE; + } + } + + static const dd_command_spec invalid_spec = { + .name = "invalid_command", + .name_len = sizeof("invalid_command") - 1, + .num_args = 1, + .outgoing_cb = _pack_invalid_command, + .incoming_cb = _process_invalid_response, + .config_features_cb = dd_command_process_config_features_unexpected, + }; + + dd_result res = dd_command_exec(conn, &invalid_spec, NULL); + if (res == dd_success) { + RETURN_TRUE; + } else { + dd_helper_close_conn(); + RETURN_FALSE; + } +} + static PHP_FUNCTION(datadog_appsec_push_addresses) { struct timespec start; - clock_gettime(CLOCK_MONOTONIC_RAW, &start); + UNUSED(clock_gettime(CLOCK_MONOTONIC_RAW, &start)); UNUSED(return_value); if (!DDAPPSEC_G(active)) { mlog(dd_log_debug, "Trying to access to push_addresses " @@ -520,18 +579,20 @@ static PHP_FUNCTION(datadog_appsec_push_addresses) return; } + struct req_exec_opts opts = {.rasp_rule = rasp_rule}; struct block_params block_params = {0}; - dd_result res = dd_request_exec(conn, addresses, rasp_rule, &block_params); + dd_result res = + dd_request_exec(conn, Z_ARRVAL_P(addresses), &opts, &block_params); - if (rasp_rule && ZSTR_LEN(rasp_rule) > 0) { + if (opts.rasp_rule && ZSTR_LEN(opts.rasp_rule) > 0) { struct timespec end; - clock_gettime(CLOCK_MONOTONIC_RAW, &end); + UNUSED(clock_gettime(CLOCK_MONOTONIC_RAW, &end)); double elapsed_us = - ((double)(end.tv_sec - start.tv_sec) * - // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) - (int64_t)1000000 + + (((double)(end.tv_sec - start.tv_sec) * + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) + (int64_t)1000000) + // NOLINTNEXTLINE(cppcoreguidelines-avoid-magic-numbers,readability-magic-numbers) - (double)(end.tv_nsec - start.tv_nsec) / 1000.0); + ((double)(end.tv_nsec - start.tv_nsec) / 1000.0)); dd_rasp_account_duration_us(elapsed_us); } @@ -544,7 +605,7 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX( ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(request_exec_arginfo, 0, 1, _IS_BOOL, 0) -ZEND_ARG_INFO(0, "data") +ZEND_ARG_TYPE_INFO(0, data, IS_ARRAY, 0) ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX( @@ -567,12 +628,20 @@ static const zend_function_entry testing_functions[] = { ZEND_RAW_FENTRY(DD_TESTING_NS "request_exec", PHP_FN(datadog_appsec_testing_request_exec), request_exec_arginfo, 0, NULL, NULL) PHP_FE_END }; +static const zend_function_entry testing_invalid_command_functions[] = { + ZEND_RAW_FENTRY(DD_TESTING_NS "send_invalid_msg", PHP_FN(datadog_appsec_testing_send_invalid_command), void_ret_bool_arginfo, 0, NULL, NULL) + PHP_FE_END +}; // clang-format on static void _register_testing_objects(void) { dd_phpobj_reg_funcs(functions); + if (get_global_DD_APPSEC_TESTING_INVALID_COMMAND()) { + dd_phpobj_reg_funcs(testing_invalid_command_functions); + } + if (!get_global_DD_APPSEC_TESTING()) { return; } diff --git a/appsec/src/extension/helper_process.c b/appsec/src/extension/helper_process.c index 90f8029afa8..60f65345475 100644 --- a/appsec/src/extension/helper_process.c +++ b/appsec/src/extension/helper_process.c @@ -6,10 +6,13 @@ // NOLINTNEXTLINE(misc-header-include-cycle) #include +#include +#include #include #include #include #include +#include #define HELPER_PROCESS_C_INCLUDES #include "compatibility.h" @@ -43,11 +46,15 @@ typedef struct _dd_helper_mgr { char *nonnull socket_path; // if abstract, starts with @ char *nonnull lock_path; // set, but not used with abstract ns sockets + char *nullable + resolved_helper_path; // resolved helper path (after redirection check) } dd_helper_mgr; static _Atomic(dd_helper_shared_state) *_shared_state; static THREAD_LOCAL_ON_ZTS dd_helper_mgr _mgr; +static THREAD_LOCAL_ON_ZTS helper_runtime _helper_runtime = + HELPER_RUNTIME_UNKNOWN; static const double _backoff_initial = 3.0; static const double _backoff_base = 2.0; @@ -62,6 +69,8 @@ static const int timeout_recv_subseq = 750; #define DD_SOCK_PATH_FORMAT DD_PATH_FORMAT ".sock" #define DD_LOCK_PATH_FORMAT DD_PATH_FORMAT ".lock" +#define RUST_HELPER_FILENAME "libddappsec-helper-rust.so" + #ifdef TESTING static void _register_testing_objects(void); #endif @@ -72,6 +81,7 @@ static bool _try_lock_shared_state(dd_helper_shared_state *nonnull s); static void _inc_failed_counter(dd_helper_shared_state *nonnull s); static void _release_shared_state_lock(dd_helper_shared_state *nonnull s); static void _maybe_reset_failed_counter(void); +static char *nullable _compute_helper_path(void); void dd_helper_startup(void) { @@ -99,6 +109,7 @@ void dd_helper_gshutdown(void) { pefree(_mgr.socket_path, 1); pefree(_mgr.lock_path, 1); + pefree(_mgr.resolved_helper_path, 1); } void dd_helper_rshutdown(void) @@ -173,6 +184,12 @@ dd_conn *nullable dd_helper_mgr_cur_conn(void) return NULL; } +helper_runtime dd_helper_get_runtime(void) { return _helper_runtime; } + +void dd_helper_set_runtime(helper_runtime rt) { _helper_runtime = rt; } + +bool dd_helper_is_rust(void) { return _helper_runtime == HELPER_RUNTIME_RUST; } + // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) bool dd_on_runtime_path_update(zval *nullable old_val, zval *nonnull new_val, zend_string *nullable new_str) @@ -220,6 +237,82 @@ static void _read_settings(void) dd_on_runtime_path_update(NULL, &runtime_path, NULL); } +// Computes the actual helper path based on DD_APPSEC_HELPER_RUST_REDIRECTION +// If redirection is enabled, looks for libddappsec-helper-rust.so alongside the +// configured helper path. Returns the path to use (allocated with pemalloc) +// or NULL to use the original path from configuration. +// Uses an empty string as a sentinel to cache negative resolutions. +static char *nullable _compute_helper_path(void) +{ + if (_mgr.resolved_helper_path) { + if (_mgr.resolved_helper_path[0] == '\0') { + return NULL; + } + return _mgr.resolved_helper_path; + } + + if (!get_global_DD_APPSEC_HELPER_RUST_REDIRECTION()) { + _mgr.resolved_helper_path = pecalloc(1, 1, 1); // empty string + return NULL; + } + + zend_string *helper_path_zs = get_DD_APPSEC_HELPER_PATH(); + const char *helper_path = ZSTR_VAL(helper_path_zs); + +#ifdef __APPLE__ + char dir_buf[PATH_MAX]; + char *dir = dirname_r(helper_path, dir_buf); + if (!dir) { + mlog(dd_log_warning, "Failed to get dirname of helper path"); + _mgr.resolved_helper_path = pecalloc(1, 1, 1); // cache negative result + return NULL; + } +#else + size_t helper_path_len = ZSTR_LEN(helper_path_zs); + char *path_copy = safe_pemalloc(helper_path_len, sizeof(char), 1, 1); + memcpy(path_copy, helper_path, helper_path_len + 1); + + // dirname is thread-safe on Linux + // NOLINTNEXTLINE(concurrency-mt-unsafe) + char *dir = dirname(path_copy); + if (!dir) { + pefree(path_copy, 1); + mlog(dd_log_warning, "Failed to get dirname of helper path"); + _mgr.resolved_helper_path = pecalloc(1, 1, 1); // cache negative result + return NULL; + } +#endif + + // Compute the Rust helper path: + // dirname(helper_path)/libddappsec-helper-rust.so + size_t dir_len = strlen(dir); + size_t rust_helper_len = sizeof(RUST_HELPER_FILENAME) - 1; + size_t total_len = dir_len + 1 + rust_helper_len; // dir + / + filename + + char *rust_helper_path = safe_pemalloc(total_len, sizeof(char), 1, 1); + snprintf( + rust_helper_path, total_len + 1, "%s/%s", dir, RUST_HELPER_FILENAME); + +#ifndef __APPLE__ + pefree(path_copy, 1); +#endif + + struct stat sb; + if (stat(rust_helper_path, &sb) == 0 && S_ISREG(sb.st_mode)) { + mlog(dd_log_debug, "Rust helper found at %s, using it", + rust_helper_path); + _mgr.resolved_helper_path = rust_helper_path; + return _mgr.resolved_helper_path; + } + + mlog(dd_log_debug, + "Rust helper not found at %s, falling back to original helper", + rust_helper_path); + pefree(rust_helper_path, 1); + _mgr.resolved_helper_path = pecalloc(1, 1, 1); // cache negative result + return NULL; +} + __attribute__((visibility("default"))) bool dd_appsec_maybe_enable_helper( sidecar_enable_appsec_t nonnull enable_appsec, // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) @@ -236,7 +329,15 @@ __attribute__((visibility("default"))) bool dd_appsec_maybe_enable_helper( _read_settings(); - ddog_CharSlice helper_path = to_char_slice(get_DD_APPSEC_HELPER_PATH()); + // Determine actual helper path (may be redirected to Rust helper) + ddog_CharSlice helper_path; + char *resolved_path = _compute_helper_path(); + if (resolved_path) { + helper_path = (ddog_CharSlice){ + .ptr = resolved_path, .len = strlen(resolved_path)}; + } else { + helper_path = to_char_slice(get_DD_APPSEC_HELPER_PATH()); + } mlog(dd_log_debug, "Helper path is %.*s", (int)helper_path.len, helper_path.ptr); ddog_CharSlice socket_path = {_mgr.socket_path, strlen(_mgr.socket_path)}; @@ -267,6 +368,7 @@ void dd_helper_close_conn(void) mlog_err(dd_log_warning, "Error closing connection to helper"); } + dd_helper_set_runtime(HELPER_RUNTIME_UNKNOWN); dd_telemetry_helper_conn_close(); /* we treat closing the connection on the request it was opened a failure diff --git a/appsec/src/extension/helper_process.h b/appsec/src/extension/helper_process.h index 35e4c7bcb12..66136696079 100644 --- a/appsec/src/extension/helper_process.h +++ b/appsec/src/extension/helper_process.h @@ -7,11 +7,22 @@ #define DD_HELPER_MGR_H #include +#include #include "attributes.h" #include "dddefs.h" #include "network.h" +typedef enum _helper_runtime { + HELPER_RUNTIME_UNKNOWN = 0, + HELPER_RUNTIME_CPP = 1, + HELPER_RUNTIME_RUST = 2, +} helper_runtime; + +helper_runtime dd_helper_get_runtime(void); +void dd_helper_set_runtime(helper_runtime rt); +bool dd_helper_is_rust(void); + typedef typeof(&ddog_sidecar_enable_appsec) sidecar_enable_appsec_t; __attribute__((visibility("default"))) bool dd_appsec_maybe_enable_helper( diff --git a/appsec/src/extension/tags.c b/appsec/src/extension/tags.c index 6913f3d08f1..e535a628822 100644 --- a/appsec/src/extension/tags.c +++ b/appsec/src/extension/tags.c @@ -9,7 +9,7 @@ #include "ddappsec.h" #include "ddtrace.h" #include "ext/pcre/php_pcre.h" -#include "ip_extraction.h" +#include "helper_process.h" #include "logging.h" #include "php_compat.h" #include "php_helpers.h" @@ -33,6 +33,7 @@ #define DD_TAG_EVENT "appsec.event" #define DD_TAG_BLOCKED "appsec.blocked" #define DD_TAG_RUNTIME_FAMILY "_dd.runtime_family" +#define DD_TAG_HELPER_RUNTIME "_dd.appsec.helper_runtime" #define DD_TAG_HTTP_METHOD "http.method" #define DD_TAG_HTTP_USER_AGENT "http.useragent" #define DD_TAG_HTTP_STATUS_CODE "http.status_code" @@ -123,6 +124,7 @@ static zend_string *_dd_login_failure_event_auto_mode; static zend_string *_dd_signup_event_sdk; static zend_string *_dd_login_success_event_sdk; static zend_string *_dd_login_failure_event_sdk; +static zend_string *_dd_tag_helper_runtime; static zend_string *_key_request_uri_zstr; static zend_string *_key_http_host_zstr; static zend_string *_key_server_name_zstr; @@ -150,7 +152,8 @@ static void _add_basic_ancillary_tags(zend_object *nonnull span, const zend_array *nonnull server, HashTable *headers); static bool _add_all_ancillary_tags( zend_object *nonnull span, const zend_array *nonnull server); -void _set_runtime_family(zend_object *nonnull span); +static void _set_runtime_family(zend_object *nonnull span); +static void _set_helper_runtime(zend_object *nonnull span); static bool _set_appsec_enabled(zval *metrics_zv); static void _register_functions(void); static void _register_test_functions(void); @@ -199,6 +202,8 @@ void dd_tags_startup(void) zend_string_init_interned(LSTRARG(DD_TAG_HTTP_RH_CONTENT_LANGUAGE), 1); _dd_tag_user = zend_string_init_interned(LSTRARG(DD_TAG_USER), 1); _dd_tag_user_id = zend_string_init_interned(LSTRARG(DD_TAG_USER_ID), 1); + _dd_tag_helper_runtime = + zend_string_init_interned(LSTRARG(DD_TAG_HELPER_RUNTIME), 1); _dd_metric_enabled = zend_string_init_interned(LSTRARG(DD_METRIC_ENABLED), 1); @@ -355,6 +360,7 @@ void dd_tags_rinit(void) void dd_tags_add_appsec_json_frag(zend_string *nonnull zstr) { + // NOLINTNEXTLINE(bugprone-multi-level-implicit-pointer-conversion) zend_llist_add_element(&_appsec_json_frags, &zstr); } @@ -391,6 +397,8 @@ void dd_tags_add_tags( } // tag _dd.runtime_family _set_runtime_family(span); + // tag _dd.appsec.helper_runtime (only if Rust) + _set_helper_runtime(span); if (zend_llist_count(&_appsec_json_frags) == 0) { if (!server) { @@ -443,6 +451,7 @@ static void _zend_string_release_indirect(void *s) zend_string_release(*(zend_string **)s); } +// NOLINTBEGIN(bugprone-multi-level-implicit-pointer-conversion) static zend_string *_concat_json_fragments(void) { #define DD_DATA_TAG_BEFORE "{\"triggers\":[" @@ -483,6 +492,7 @@ static zend_string *_concat_json_fragments(void) return tag_value; } +// NOLINTEND(bugprone-multi-level-implicit-pointer-conversion) static void _add_basic_tags_to_meta( zval *nonnull meta, const zend_array *nonnull server, HashTable *headers); @@ -857,7 +867,7 @@ static zend_string *nullable _is_relevant_resp_header( return NULL; } -void _set_runtime_family(zend_object *nonnull span) +static void _set_runtime_family(zend_object *nonnull span) { bool res = dd_trace_span_add_tag_str( span, LSTRARG(DD_TAG_RUNTIME_FAMILY), LSTRARG("php")); @@ -867,6 +877,18 @@ void _set_runtime_family(zend_object *nonnull span) } } +static void _set_helper_runtime(zend_object *nonnull span) +{ + if (dd_helper_is_rust()) { + bool res = dd_trace_span_add_tag_str( + span, LSTRARG(DD_TAG_HELPER_RUNTIME), LSTRARG("rust")); + if (!res && !get_global_DD_APPSEC_TESTING()) { + mlog(dd_log_warning, + "Failed to add " DD_TAG_HELPER_RUNTIME " to root span"); + } + } +} + static void _add_custom_event_keyval(zend_array *nonnull meta_ht, // NOLINTNEXTLINE(bugprone-easily-swappable-parameters) zend_string *nonnull event, zend_string *nonnull key, diff --git a/appsec/src/extension/telemetry.c b/appsec/src/extension/telemetry.c index 3c58a391e36..9c335d75475 100644 --- a/appsec/src/extension/telemetry.c +++ b/appsec/src/extension/telemetry.c @@ -1,6 +1,7 @@ #include "telemetry.h" #include "configuration.h" #include "ddtrace.h" +#include "helper_process.h" #include "logging.h" #include "php_compat.h" #include "string_helpers.h" @@ -57,8 +58,13 @@ static void _add_helper_conn_metric(zend_string *nonnull name_zstr) } zend_string *runtime_path = get_DD_APPSEC_HELPER_RUNTIME_PATH(); char *tags = NULL; - size_t tags_len = + if (dd_helper_is_rust()) { + spprintf(&tags, 0, "runtime_path:%s,helper_runtime:rust", + ZSTR_VAL(runtime_path)); + } else { spprintf(&tags, 0, "runtime_path:%s", ZSTR_VAL(runtime_path)); + } + size_t tags_len = strlen(tags); zend_string *tags_zstr = zend_string_init(tags, tags_len, 0); dd_telemetry_add_metric(name_zstr, 1, tags_zstr, DDTRACE_METRIC_TYPE_COUNT); zend_string_release(tags_zstr); diff --git a/appsec/src/extension/user_tracking.c b/appsec/src/extension/user_tracking.c index 3e1586967a7..2fdd3c96d03 100644 --- a/appsec/src/extension/user_tracking.c +++ b/appsec/src/extension/user_tracking.c @@ -264,14 +264,15 @@ void dd_find_and_apply_verdict_for_user(zend_string *nullable user_id, } struct block_params block_params = {0}; - dd_result res = dd_request_exec(conn, &data_zv, false, &block_params); + dd_result res = dd_request_exec( + conn, Z_ARRVAL(data_zv), &(struct req_exec_opts){0}, &block_params); if (res == dd_network) { mlog_g(dd_log_info, "request_exec failed with dd_network; closing " "connection to helper"); dd_helper_close_conn(); } - zval_ptr_dtor(&data_zv); + zend_array_destroy(Z_ARRVAL(data_zv)); if (user_id != NULL && ZSTR_LEN(user_id) > 0) { dd_tags_set_event_user_id(user_id); diff --git a/appsec/src/helper/client.cpp b/appsec/src/helper/client.cpp index 785736d650b..5a524c60ee2 100644 --- a/appsec/src/helper/client.cpp +++ b/appsec/src/helper/client.cpp @@ -202,6 +202,8 @@ bool client::handle_command(const network::client_init::request &command) collect_metrics(*response, *service_, context_, sc_settings_); } + response->helper_runtime = "cpp"; + try { if (!broker_->send(response)) { has_errors = true; @@ -232,14 +234,14 @@ template bool client::service_guard() template std::shared_ptr client::publish( - typename T::request &command, const std::string &rasp_rule) + typename T::request &command, const network::request_exec_options &options) { SPDLOG_DEBUG("received command {}", T::name); auto response = std::make_shared(); try { // NOLINTNEXTLINE(bugprone-unchecked-optional-access) - auto res = context_->publish(std::move(command.data), rasp_rule); + auto res = context_->publish(std::move(command.data), options); if (res) { bool event_action = false; bool stack_trace = false; @@ -321,7 +323,7 @@ bool client::handle_command(network::request_init::request &command) // During request init we initialize the engine context context_.emplace(*service_->get_engine()); - auto response = publish(command); + auto response = publish(command, {}); if (response) { response->settings["auto_user_instrum"] = to_string_view( service_->get_service_config()->get_auto_user_intrum_mode()); @@ -340,7 +342,7 @@ bool client::handle_command(network::request_exec::request &command) context_.emplace(*service_->get_engine()); } - auto response = publish(command, command.rasp_rule); + auto response = publish(command, command.options); return send_message(response); } @@ -467,7 +469,7 @@ bool client::handle_command(network::request_shutdown::request &command) command.data.add("waf.context.processor", std::move(context_processor)); } - auto response = publish(command); + auto response = publish(command, {}); if (!response) { return false; } diff --git a/appsec/src/helper/client.hpp b/appsec/src/helper/client.hpp index c4473949d9b..e6d04132a8f 100644 --- a/appsec/src/helper/client.hpp +++ b/appsec/src/helper/client.hpp @@ -130,8 +130,8 @@ class client { } template - std::shared_ptr publish( - typename T::request &command, const std::string &rasp_rule = ""); + std::shared_ptr publish(typename T::request &command, + const network::request_exec_options &options); template bool service_guard(); template bool send_message(const std::shared_ptr &message); diff --git a/appsec/src/helper/engine.cpp b/appsec/src/helper/engine.cpp index 14a80f6710e..083839ebdaf 100644 --- a/appsec/src/helper/engine.cpp +++ b/appsec/src/helper/engine.cpp @@ -82,7 +82,7 @@ void engine::update(const rapidjson::Document &doc, } std::optional engine::context::publish( - parameter &¶m, const std::string &rasp_rule) + parameter &¶m, const network::request_exec_options &options) { // Once the parameter reaches this function, it is guaranteed to be // owned by the engine. @@ -113,7 +113,7 @@ std::optional engine::context::publish( } try { const auto &listener = it->second; - listener->call(data, event, rasp_rule); + listener->call(data, event, options); } catch (std::exception &e) { SPDLOG_ERROR("subscriber failed: {}", e.what()); } diff --git a/appsec/src/helper/engine.hpp b/appsec/src/helper/engine.hpp index 4ed11ba7a5a..974f31562ce 100644 --- a/appsec/src/helper/engine.hpp +++ b/appsec/src/helper/engine.hpp @@ -7,6 +7,7 @@ #include "action.hpp" #include "engine_settings.hpp" +#include "network/proto.hpp" #include "parameter.hpp" #include "rate_limit.hpp" #include "subscriber/base.hpp" @@ -53,11 +54,14 @@ class engine { // store a shared_ptr to the engine class context { public: +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" explicit context(engine &engine) : common_{std::atomic_load_explicit( &engine.common_, std::memory_order_acquire)}, limiter_{engine.limiter_} {} +#pragma GCC diagnostic pop context(const context &) = delete; context &operator=(const context &) = delete; context(context &&) = delete; @@ -65,7 +69,7 @@ class engine { ~context() = default; std::optional publish( - parameter &¶m, const std::string &rasp_rule = ""); + parameter &¶m, const network::request_exec_options &options); // NOLINTNEXTLINE(google-runtime-references) void get_metrics(telemetry::telemetry_submitter &msubmitter); [[nodiscard]] bool get_input_truncated() const diff --git a/appsec/src/helper/network/proto.hpp b/appsec/src/helper/network/proto.hpp index e27e13b27c1..c8e91cf42df 100644 --- a/appsec/src/helper/network/proto.hpp +++ b/appsec/src/helper/network/proto.hpp @@ -137,8 +137,9 @@ struct client_init { std::map meta; std::map metrics; + std::optional helper_runtime; - MSGPACK_DEFINE(status, version, errors, meta, metrics) + MSGPACK_DEFINE(status, version, errors, meta, metrics, helper_runtime) }; }; @@ -185,6 +186,14 @@ struct request_init { }; }; +struct request_exec_options { + std::optional rasp_rule; + std::optional subctx_id; + std::optional subctx_last_call; + + MSGPACK_DEFINE_MAP(rasp_rule, subctx_id, subctx_last_call) +}; + struct request_exec { static constexpr const char *name = "request_exec"; @@ -192,8 +201,8 @@ struct request_exec { static constexpr const char *name = request_exec::name; static constexpr request_id id = request_id::request_exec; - std::string rasp_rule; dds::parameter data; + request_exec_options options; request() = default; request(const request &) = delete; @@ -202,7 +211,7 @@ struct request_exec { request &operator=(request &&) = default; ~request() override = default; - MSGPACK_DEFINE(rasp_rule, data) + MSGPACK_DEFINE(data, options) }; struct response : base_response_generic { diff --git a/appsec/src/helper/subscriber/base.hpp b/appsec/src/helper/subscriber/base.hpp index d2da71b43bf..29871b95fb8 100644 --- a/appsec/src/helper/subscriber/base.hpp +++ b/appsec/src/helper/subscriber/base.hpp @@ -6,6 +6,7 @@ #pragma once #include "../action.hpp" +#include "../network/proto.hpp" #include "../parameter_view.hpp" #include "../remote_config/changeset.hpp" #include "../telemetry.hpp" @@ -26,7 +27,7 @@ class subscriber { virtual ~listener() = default; // NOLINTNEXTLINE(google-runtime-references) virtual void call(parameter_view &data, event &event, - const std::string &rasp_rule = "") = 0; + const network::request_exec_options &options) = 0; // NOLINTNEXTLINE(google-runtime-references) virtual void submit_metrics( diff --git a/appsec/src/helper/subscriber/waf.cpp b/appsec/src/helper/subscriber/waf.cpp index f19fa41e04c..74b9075f95b 100644 --- a/appsec/src/helper/subscriber/waf.cpp +++ b/appsec/src/helper/subscriber/waf.cpp @@ -541,9 +541,10 @@ instance::listener::~listener() } } -void instance::listener::call( - dds::parameter_view &data, event &event, const std::string &rasp_rule) +void instance::listener::call(dds::parameter_view &data, event &event, + const network::request_exec_options &options) { + const auto &rasp_rule = options.rasp_rule.value_or(""); ddwaf_object res; DDWAF_RET_CODE code; unsigned duration = 0; diff --git a/appsec/src/helper/subscriber/waf.hpp b/appsec/src/helper/subscriber/waf.hpp index aa2da9b78b9..c258641c95f 100644 --- a/appsec/src/helper/subscriber/waf.hpp +++ b/appsec/src/helper/subscriber/waf.hpp @@ -47,7 +47,7 @@ class instance : public dds::subscriber { ~listener() override; void call(dds::parameter_view &data, event &event, - const std::string &rasp_rule) override; + const network::request_exec_options &options) override; // NOLINTNEXTLINE(google-runtime-references) void submit_metrics( diff --git a/appsec/tests/extension/actions_handling_01.phpt b/appsec/tests/extension/actions_handling_01.phpt index 71fa9508192..1007743e101 100644 --- a/appsec/tests/extension/actions_handling_01.phpt +++ b/appsec/tests/extension/actions_handling_01.phpt @@ -32,8 +32,6 @@ array(2) { [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(1) { ["server.request.path_params"]=> array(2) { @@ -43,5 +41,8 @@ array(2) { string(10) "parameters" } } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/client_init_fail.phpt b/appsec/tests/extension/client_init_fail.phpt index b26a43685c8..0b5faeb8599 100644 --- a/appsec/tests/extension/client_init_fail.phpt +++ b/appsec/tests/extension/client_init_fail.phpt @@ -10,7 +10,7 @@ $obj = new ArrayObject(); $helper = Helper::createRun( [ response_list( - response_client_init(['not-ok', phpversion('ddappsec'), ['such and such error occurred'], $obj, $obj]) + response_client_init(['not-ok', phpversion('ddappsec'), ['such and such error occurred'], $obj, $obj, null]) ) ]); diff --git a/appsec/tests/extension/client_init_record_span_tags.phpt b/appsec/tests/extension/client_init_record_span_tags.phpt index 73d78d5769c..5b4a68db1f7 100644 --- a/appsec/tests/extension/client_init_record_span_tags.phpt +++ b/appsec/tests/extension/client_init_record_span_tags.phpt @@ -32,7 +32,7 @@ $helper = Helper::createRun([ response_list( response_client_init(['ok', phpversion('ddappsec'), [], ["meta_1" => "value_1", "meta_2" => "value_2"], - ["metric_1" => 2.0, "metric_2" => 10.0]]) + ["metric_1" => 2.0, "metric_2" => 10.0], null]) ), response_list( response_request_init([[['record', []]], ['{"found":"attack"}','{"another":"attack"}']]) diff --git a/appsec/tests/extension/client_init_wrong_version.phpt b/appsec/tests/extension/client_init_wrong_version.phpt index 940595312c2..b2005698c66 100644 --- a/appsec/tests/extension/client_init_wrong_version.phpt +++ b/appsec/tests/extension/client_init_wrong_version.phpt @@ -9,7 +9,7 @@ include __DIR__ . '/inc/mock_helper.php'; $obj = new ArrayObject(); $helper = Helper::createRun([ response_list( - response_client_init(['ok', '0.0.0',[],$obj,$obj]) + response_client_init(['ok', '0.0.0',[],$obj,$obj,null]) ) ]); diff --git a/appsec/tests/extension/inc/mock_helper.php b/appsec/tests/extension/inc/mock_helper.php index d7abd2fe075..918a9a32326 100644 --- a/appsec/tests/extension/inc/mock_helper.php +++ b/appsec/tests/extension/inc/mock_helper.php @@ -51,7 +51,7 @@ static function createInitedRun($responses, $opts = array()) { $empty_obj = new ArrayObject(); $responses = array_merge([ response_list( - response_client_init(['ok', phpversion('ddappsec'),[],$empty_obj,$empty_obj]) + response_client_init(['ok', phpversion('ddappsec'),[],$empty_obj,$empty_obj,null]) )], $responses); return self::createRun($responses, $opts); } diff --git a/appsec/tests/extension/input_truncated_05.phpt b/appsec/tests/extension/input_truncated_05.phpt index d814bd9e571..fb42f4e9464 100644 --- a/appsec/tests/extension/input_truncated_05.phpt +++ b/appsec/tests/extension/input_truncated_05.phpt @@ -41,16 +41,16 @@ rshutdown(); $commands = $helper->get_commands(); -$test1_data = $commands[2][1][1]; +$test1_data = $commands[2][1][0]; // outer array + container + 2046 echo "test1: number of elements inside: ", count(reset($test1_data)), "\n"; -$test2_data = $commands[3][1][1]; +$test2_data = $commands[3][1][0]; // outer array + container + 2000 + container + 44 echo "test2: number of elements inside 1st array: ", count($test2_data['test2'][0]), "\n"; echo "test2: number of elements inside 2st array: ", count($test2_data['test2'][1]), "\n"; -$test3_data = $commands[4][1][1]; +$test3_data = $commands[4][1][0]; $d = $test3_data['test3']; $num_arrs = count(array_filter($d, function ($x) { return $x === array(1); })); $num_nils = count(array_filter($d, function ($x) { return $x === null; })); diff --git a/appsec/tests/extension/push_params_ok_01.phpt b/appsec/tests/extension/push_params_ok_01.phpt index 9f1725b2679..e0feb991152 100644 --- a/appsec/tests/extension/push_params_ok_01.phpt +++ b/appsec/tests/extension/push_params_ok_01.phpt @@ -32,8 +32,6 @@ array(2) { [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(1) { ["server.request.path_params"]=> array(2) { @@ -43,5 +41,8 @@ array(2) { string(10) "parameters" } } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/push_params_ok_02.phpt b/appsec/tests/extension/push_params_ok_02.phpt index 6de13648ccd..592bc0abfa3 100644 --- a/appsec/tests/extension/push_params_ok_02.phpt +++ b/appsec/tests/extension/push_params_ok_02.phpt @@ -32,11 +32,12 @@ array(2) { [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(1) { ["server.request.path_params"]=> string(11) "some string" } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/push_params_ok_03.phpt b/appsec/tests/extension/push_params_ok_03.phpt index 0746394c58c..18ec5af0d7f 100644 --- a/appsec/tests/extension/push_params_ok_03.phpt +++ b/appsec/tests/extension/push_params_ok_03.phpt @@ -32,11 +32,12 @@ array(2) { [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(1) { ["server.request.path_params"]=> int(1234) } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/push_params_ok_04.phpt b/appsec/tests/extension/push_params_ok_04.phpt index 9630daa5bfa..41fa0f5c2d6 100644 --- a/appsec/tests/extension/push_params_ok_04.phpt +++ b/appsec/tests/extension/push_params_ok_04.phpt @@ -39,11 +39,14 @@ array(2) { [1]=> array(2) { [0]=> - string(3) "lfi" - [1]=> array(1) { ["server.request.path_params"]=> int(1234) } + [1]=> + array(1) { + ["rasp_rule"]=> + string(3) "lfi" + } } } diff --git a/appsec/tests/extension/push_params_ok_07.phpt b/appsec/tests/extension/push_params_ok_07.phpt index ea8343ecb85..c9705560f5a 100644 --- a/appsec/tests/extension/push_params_ok_07.phpt +++ b/appsec/tests/extension/push_params_ok_07.phpt @@ -32,8 +32,6 @@ array(2) { [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(2) { ["server.request.path_params"]=> array(2) { @@ -45,5 +43,8 @@ array(2) { ["some.other"]=> int(12345) } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/push_params_ok_08.phpt b/appsec/tests/extension/push_params_ok_08.phpt index cfd8a6727c8..285c4d9fac2 100644 --- a/appsec/tests/extension/push_params_ok_08.phpt +++ b/appsec/tests/extension/push_params_ok_08.phpt @@ -32,8 +32,6 @@ array(2) { [1]=> array(2) { [0]=> - string(3) "lfi" - [1]=> array(1) { ["server.request.path_params"]=> array(2) { @@ -43,5 +41,10 @@ array(2) { string(10) "parameters" } } + [1]=> + array(1) { + ["rasp_rule"]=> + string(3) "lfi" + } } } diff --git a/appsec/tests/extension/push_params_ok_09.phpt b/appsec/tests/extension/push_params_ok_09.phpt index 8e03d406924..7b8223dd644 100644 --- a/appsec/tests/extension/push_params_ok_09.phpt +++ b/appsec/tests/extension/push_params_ok_09.phpt @@ -32,8 +32,6 @@ array(2) { [1]=> array(2) { [0]=> - string(4) "ssrf" - [1]=> array(1) { ["server.request.path_params"]=> array(2) { @@ -43,5 +41,10 @@ array(2) { string(10) "parameters" } } + [1]=> + array(1) { + ["rasp_rule"]=> + string(4) "ssrf" + } } } diff --git a/appsec/tests/extension/request_exec.phpt b/appsec/tests/extension/request_exec.phpt index 8071b55d320..a1a6a450c49 100644 --- a/appsec/tests/extension/request_exec.phpt +++ b/appsec/tests/extension/request_exec.phpt @@ -26,10 +26,6 @@ var_dump(request_exec([ 'key 03' => ['some' => 'array'] ])); -var_dump(request_exec('value')); -var_dump(request_exec(55)); - - rshutdown(); $commands = $helper->get_commands(); @@ -39,16 +35,12 @@ var_dump($commands[2]); ?> --EXPECTF-- bool(true) -bool(false) -bool(false) array(2) { [0]=> string(12) "request_exec" [1]=> array(2) { [0]=> - string(0) "" - [1]=> array(3) { ["key 01"]=> string(10) "some value" @@ -60,5 +52,8 @@ array(2) { string(5) "array" } } + [1]=> + array(0) { + } } } diff --git a/appsec/tests/extension/user_tracking_do_nothing_from_login_success.phpt b/appsec/tests/extension/user_tracking_do_nothing_from_login_success.phpt index 1df24b27de4..2e8ce95ba11 100644 --- a/appsec/tests/extension/user_tracking_do_nothing_from_login_success.phpt +++ b/appsec/tests/extension/user_tracking_do_nothing_from_login_success.phpt @@ -32,7 +32,7 @@ track_user_login_success_event("Admin", $c = $helper->get_commands(); echo "usr.id:\n"; -var_dump($c[0][1][1]['usr.id']); +var_dump($c[0][1][0]['usr.id']); echo "root_span_get_meta():\n"; print_r(root_span_get_meta()); diff --git a/appsec/tests/extension/user_tracking_do_nothing_from_login_success_v2.phpt b/appsec/tests/extension/user_tracking_do_nothing_from_login_success_v2.phpt index 6d579804160..5ec67472f24 100644 --- a/appsec/tests/extension/user_tracking_do_nothing_from_login_success_v2.phpt +++ b/appsec/tests/extension/user_tracking_do_nothing_from_login_success_v2.phpt @@ -31,7 +31,7 @@ $helper->get_commands(); // Ignore $c = $helper->get_commands(); echo "usr.id:\n"; -var_dump($c[0][1][1]['usr.id']); +var_dump($c[0][1][0]['usr.id']); echo "root_span_get_meta():\n"; print_r(root_span_get_meta()); diff --git a/appsec/tests/extension/user_tracking_do_nothing_from_set_user.phpt b/appsec/tests/extension/user_tracking_do_nothing_from_set_user.phpt index 871091d356d..fe909f198ec 100644 --- a/appsec/tests/extension/user_tracking_do_nothing_from_set_user.phpt +++ b/appsec/tests/extension/user_tracking_do_nothing_from_set_user.phpt @@ -30,7 +30,7 @@ DDTrace\set_user("Admin", $c = $helper->get_commands(); echo "usr.id:\n"; -var_dump($c[0][1][1]['usr.id']); +var_dump($c[0][1][0]['usr.id']); echo "root_span_get_meta():\n"; print_r(root_span_get_meta()); diff --git a/appsec/tests/helper/broker_test.cpp b/appsec/tests/helper/broker_test.cpp index 3d193c21702..761dec3dee7 100644 --- a/appsec/tests/helper/broker_test.cpp +++ b/appsec/tests/helper/broker_test.cpp @@ -80,7 +80,7 @@ TEST(BrokerTest, SendClientInit) packer.pack_array(1); // Array of messages packer.pack_array(2); // First message pack_str(packer, "client_init"); // Type - packer.pack_array(5); + packer.pack_array(6); pack_str(packer, "ok"); pack_str(packer, dds::php_ddappsec_version); packer.pack_array(2); @@ -88,6 +88,7 @@ TEST(BrokerTest, SendClientInit) pack_str(packer, "two"); packer.pack_map(0); packer.pack_map(0); + pack_str(packer, "cpp"); // helper_runtime const auto &expected_data = ss.str(); network::header_t h; @@ -100,6 +101,7 @@ TEST(BrokerTest, SendClientInit) auto response = std::make_shared(); response->status = "ok"; response->errors = {"one", "two"}; + response->helper_runtime = "cpp"; std::vector> messages; messages.push_back(response); diff --git a/appsec/tests/helper/client_test.cpp b/appsec/tests/helper/client_test.cpp index d51382be425..61e47ae4c99 100644 --- a/appsec/tests/helper/client_test.cpp +++ b/appsec/tests/helper/client_test.cpp @@ -190,6 +190,8 @@ TEST(ClientTest, ClientInit) ddwaf_get_version()); EXPECT_STREQ( msg_res->meta[std::string(metrics::event_rules_errors)].c_str(), "{}"); + EXPECT_TRUE(msg_res->helper_runtime.has_value()); + EXPECT_STREQ(msg_res->helper_runtime->c_str(), "cpp"); EXPECT_EQ(msg_res->metrics.size(), 2); EXPECT_EQ(msg_res->metrics[metrics::event_rules_loaded], 5.0); @@ -297,6 +299,8 @@ TEST(ClientTest, ClientInitInvalidRules) EXPECT_EQ(msg_res->meta.size(), 2); EXPECT_STREQ(msg_res->meta[std::string(metrics::waf_version)].c_str(), ddwaf_get_version()); + EXPECT_TRUE(msg_res->helper_runtime.has_value()); + EXPECT_STREQ(msg_res->helper_runtime->c_str(), "cpp"); rapidjson::Document doc; doc.Parse(msg_res->meta[std::string(metrics::event_rules_errors)]); @@ -2957,7 +2961,7 @@ TEST(ClientTest, RaspCalls) { network::request_exec::request msg; - msg.rasp_rule = "lfi"; + msg.options.rasp_rule = "lfi"; msg.data = parameter::map(); network::request req(std::move(msg)); diff --git a/appsec/tests/helper/engine_test.cpp b/appsec/tests/helper/engine_test.cpp index 8ab78fbcffd..3436374aa87 100644 --- a/appsec/tests/helper/engine_test.cpp +++ b/appsec/tests/helper/engine_test.cpp @@ -31,8 +31,8 @@ namespace mock { class listener : public dds::subscriber::listener { public: MOCK_METHOD1(submit_metrics, void(telemetry::telemetry_submitter &)); - MOCK_METHOD3( - call, void(dds::parameter_view &, dds::event &, const std::string &)); + MOCK_METHOD3(call, void(dds::parameter_view &, dds::event &, + const dds::network::request_exec_options &)); MOCK_METHOD2( get_meta_and_metrics, void(std::map &, std::map &)); @@ -55,7 +55,7 @@ TEST(EngineTest, NoSubscriptors) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -67,9 +67,9 @@ TEST(EngineTest, SingleSubscriptor) EXPECT_CALL(*sub, get_listener()).WillRepeatedly(Invoke([]() { auto listener = std::make_unique(); EXPECT_CALL(*listener, call(_, _, _)) - .WillRepeatedly( - Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { + .WillRepeatedly(Invoke( + [](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { event_.actions.push_back({dds::action_type::block, {}}); })); return listener; @@ -81,13 +81,13 @@ TEST(EngineTest, SingleSubscriptor) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("b", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); } @@ -100,24 +100,26 @@ TEST(EngineTest, MultipleSubscriptors) auto blocker = std::make_unique(); EXPECT_CALL(*blocker, call(_, _, _)) - .WillRepeatedly(Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { - std::unordered_set subs{"a", "b", "e", "f"}; - if (subs.find(data[0].parameterName) != subs.end()) { - event_.triggers.push_back("some event"); - event_.actions.push_back({dds::action_type::block, {}}); - } - })); + .WillRepeatedly(Invoke( + [](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { + std::unordered_set subs{"a", "b", "e", "f"}; + if (subs.find(data[0].parameterName) != subs.end()) { + event_.triggers.push_back("some event"); + event_.actions.push_back({dds::action_type::block, {}}); + } + })); auto recorder = std::make_unique(); EXPECT_CALL(*recorder, call(_, _, _)) - .WillRepeatedly(Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { - std::unordered_set subs{"c", "d", "e", "g"}; - if (subs.find(data[0].parameterName) != subs.end()) { - event_.triggers.push_back("some event"); - } - })); + .WillRepeatedly(Invoke( + [](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { + std::unordered_set subs{"c", "d", "e", "g"}; + if (subs.find(data[0].parameterName) != subs.end()) { + event_.triggers.push_back("some event"); + } + })); std::unique_ptr ignorer = std::unique_ptr(new mock::listener()); @@ -149,63 +151,63 @@ TEST(EngineTest, MultipleSubscriptors) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("b", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("c", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); p = parameter::map(); p.add("d", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); p = parameter::map(); p.add("e", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("f", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("g", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); p = parameter::map(); p.add("h", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); p = parameter::map(); p.add("a", parameter::string("value"sv)); p.add("c", parameter::string("value"sv)); p.add("h", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("c", parameter::string("value"sv)); p.add("h", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -221,9 +223,9 @@ TEST(EngineTest, StatefulSubscriptor) auto listener = std::make_unique(); EXPECT_CALL(*listener, call(_, _, _)) .Times(3) - .WillRepeatedly( - Invoke([&attempt](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { + .WillRepeatedly(Invoke( + [&attempt](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { if (attempt == 2 || attempt == 5) { event_.actions.push_back({dds::action_type::block, {}}); } @@ -238,17 +240,17 @@ TEST(EngineTest, StatefulSubscriptor) parameter p = parameter::map(); p.add("sub1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); p = parameter::map(); p.add("sub2", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); p = parameter::map(); p.add("final", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); @@ -256,17 +258,17 @@ TEST(EngineTest, StatefulSubscriptor) p = parameter::map(); p.add("final", parameter::string("value"sv)); - res = ctx2.publish(std::move(p)); + res = ctx2.publish(std::move(p), {}); EXPECT_FALSE(res); p = parameter::map(); p.add("sub1", parameter::string("value"sv)); - res = ctx2.publish(std::move(p)); + res = ctx2.publish(std::move(p), {}); EXPECT_FALSE(res); p = parameter::map(); p.add("sub2", parameter::string("value"sv)); - res = ctx2.publish(std::move(p)); + res = ctx2.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); } @@ -278,7 +280,8 @@ TEST(EngineTest, WafDefaultActions) auto listener = std::make_unique(); EXPECT_CALL(*listener, call(_, _, _)) .WillRepeatedly(Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { + const dds::network::request_exec_options + &options) -> void { event_.actions.push_back({dds::action_type::redirect, {}}); event_.actions.push_back({dds::action_type::block, {}}); event_.actions.push_back({dds::action_type::stack_trace, {}}); @@ -296,7 +299,7 @@ TEST(EngineTest, WafDefaultActions) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(4, res->actions.size()); EXPECT_EQ(res->actions[0].type, dds::action_type::redirect); @@ -306,7 +309,7 @@ TEST(EngineTest, WafDefaultActions) p = parameter::map(); p.add("b", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(4, res->actions.size()); EXPECT_EQ(res->actions[0].type, dds::action_type::redirect); @@ -321,11 +324,12 @@ TEST(EngineTest, InvalidActionsAreDiscarded) auto listener = std::make_unique(); EXPECT_CALL(*listener, call(_, _, _)) - .WillRepeatedly(Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { - event_.actions.push_back({dds::action_type::invalid, {}}); - event_.actions.push_back({dds::action_type::block, {}}); - })); + .WillRepeatedly(Invoke( + [](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { + event_.actions.push_back({dds::action_type::invalid, {}}); + event_.actions.push_back({dds::action_type::block, {}}); + })); auto sub = std::make_unique(); EXPECT_CALL(*sub, get_listener()).WillOnce(Invoke([&]() { @@ -338,14 +342,14 @@ TEST(EngineTest, InvalidActionsAreDiscarded) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(1, res->actions.size()); EXPECT_EQ(res->actions[0].type, dds::action_type::block); p = parameter::map(); p.add("b", parameter::string("value"sv)); - res = ctx.publish(std::move(p)); + res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(1, res->actions.size()); EXPECT_EQ(res->actions[0].type, dds::action_type::block); @@ -378,7 +382,7 @@ TEST(EngineTest, WafSubscriptorBasic) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); Mock::VerifyAndClearExpectations(&msubmitter); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); @@ -402,7 +406,7 @@ TEST(EngineTest, WafSubscriptorInvalidParam) auto p = parameter::array(); - EXPECT_THROW(ctx.publish(std::move(p)), invalid_object); + EXPECT_THROW(ctx.publish(std::move(p), {}), invalid_object); } TEST(EngineTest, WafSubscriptorTimeout) @@ -418,7 +422,7 @@ TEST(EngineTest, WafSubscriptorTimeout) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -469,7 +473,7 @@ TEST(EngineTest, MockSubscriptorsUpdateRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -514,7 +518,7 @@ TEST(EngineTest, MockSubscriptorsInvalidRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -531,7 +535,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -559,7 +563,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); EXPECT_EQ(res->triggers.size(), 1); @@ -589,7 +593,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } } @@ -607,7 +611,7 @@ TEST(EngineTest, WafSubscriptorInvalidRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -634,7 +638,7 @@ TEST(EngineTest, WafSubscriptorInvalidRuleData) auto p = parameter::map(); p.add("http.client_ip", parameter::string("192.168.1.1"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } } @@ -652,7 +656,7 @@ TEST(EngineTest, WafSubscriptorUpdateRules) auto p = parameter::map(); p.add("server.request.query", parameter::string("/some-url"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -669,7 +673,7 @@ TEST(EngineTest, WafSubscriptorUpdateRules) auto p = parameter::map(); p.add("server.request.query", parameter::string("/some-url"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); EXPECT_EQ(res->triggers.size(), 1); @@ -690,7 +694,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverride) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -709,7 +713,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverride) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -726,7 +730,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverride) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } } @@ -745,7 +749,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -767,7 +771,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::redirect); } @@ -787,7 +791,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -798,7 +802,7 @@ TEST(EngineTest, WafSubscriptorUpdateRuleOverrideAndActions) auto p = parameter::map(); p.add("arg4", parameter::string("string 4"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); EXPECT_EQ(res->force_keep, true); @@ -818,7 +822,7 @@ TEST(EngineTest, TestKeep) auto p = parameter::map(); p.add("arg12", parameter::string("string 12"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } { @@ -827,7 +831,7 @@ TEST(EngineTest, TestKeep) auto p = parameter::map(); p.add("arg5", parameter::string("string 5"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } { @@ -836,7 +840,7 @@ TEST(EngineTest, TestKeep) auto p = parameter::map(); p.add("arg4", parameter::string("string 4"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); EXPECT_EQ(res->force_keep, true); @@ -857,7 +861,7 @@ TEST(EngineTest, WafSubscriptorExclusions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -877,7 +881,7 @@ TEST(EngineTest, WafSubscriptorExclusions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -894,7 +898,7 @@ TEST(EngineTest, WafSubscriptorExclusions) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } } @@ -911,7 +915,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -923,7 +927,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -943,7 +947,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); ASSERT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); } @@ -956,7 +960,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); ASSERT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -974,7 +978,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -986,7 +990,7 @@ TEST(EngineTest, WafSubscriptorCustomRules) p.add("arg1", parameter::string("string 1"sv)); p.add("arg2", parameter::string("string 3"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); ASSERT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -1002,11 +1006,12 @@ TEST(EngineTest, RateLimiterDoNotForceKeep) EXPECT_CALL(*sub, get_listener()).WillRepeatedly(Invoke([&]() { auto listener = std::make_unique(); EXPECT_CALL(*listener, call(_, _, _)) - .WillOnce(Invoke([](dds::parameter_view &data, dds::event &event_, - std::string rasp) -> void { - event_.actions.push_back({dds::action_type::redirect, {}}); - event_.keep = false; - })); + .WillOnce(Invoke( + [](dds::parameter_view &data, dds::event &event_, + const dds::network::request_exec_options &options) -> void { + event_.actions.push_back({dds::action_type::redirect, {}}); + event_.keep = false; + })); return listener; })); @@ -1014,10 +1019,10 @@ TEST(EngineTest, RateLimiterDoNotForceKeep) parameter p = parameter::map(); p.add("a", parameter::string("value"sv)); - auto res = e->get_context().publish(std::move(p)); + auto res = e->get_context().publish(std::move(p), {}); parameter p2 = parameter::map(); p2.add("a", parameter::string("value"sv)); - res = e->get_context().publish(std::move(p2)); + res = e->get_context().publish(std::move(p2), {}); EXPECT_FALSE(res->force_keep); } diff --git a/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp b/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp index 5a86bebcc0e..cd98df65f88 100644 --- a/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp +++ b/appsec/tests/helper/remote_config/listeners/engine_listener_test.cpp @@ -139,7 +139,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdate) auto p = parameter::map(); p.add("server.request.query", parameter::string("/anotherUrl"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -161,7 +161,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdate) auto p = parameter::map(); p.add("server.request.query", parameter::string("/anotherUrl"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); ASSERT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); EXPECT_EQ(res->triggers.size(), 1); @@ -198,7 +198,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdateFallback) auto p = parameter::map(); p.add("server.request.query", parameter::string("/a/url"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); EXPECT_EQ(res->triggers.size(), 1); @@ -216,7 +216,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdateFallback) auto p = parameter::map(); p.add("server.request.query", parameter::string("/a/url"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -226,7 +226,7 @@ TEST(RemoteConfigEngineListener, EngineRuleUpdateFallback) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } } @@ -248,7 +248,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideUpdateDisableRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -262,7 +262,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideUpdateDisableRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -273,7 +273,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideUpdateDisableRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } } @@ -296,7 +296,7 @@ TEST(RemoteConfigEngineListener, RuleOverrideUpdateSetOnMatch) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -311,7 +311,7 @@ TEST(RemoteConfigEngineListener, RuleOverrideUpdateSetOnMatch) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -323,7 +323,7 @@ TEST(RemoteConfigEngineListener, RuleOverrideUpdateSetOnMatch) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); } @@ -347,7 +347,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideAndActionsUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -364,7 +364,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideAndActionsUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::record); } @@ -376,7 +376,7 @@ TEST(RemoteConfigEngineListener, EngineRuleOverrideAndActionsUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::redirect); } @@ -400,7 +400,7 @@ TEST(RemoteConfigEngineListener, EngineExclusionsUpdatePasslistRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -414,7 +414,7 @@ TEST(RemoteConfigEngineListener, EngineExclusionsUpdatePasslistRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -425,7 +425,7 @@ TEST(RemoteConfigEngineListener, EngineExclusionsUpdatePasslistRule) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } } @@ -448,7 +448,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -458,7 +458,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -476,7 +476,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -486,7 +486,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -497,7 +497,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -507,7 +507,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -522,7 +522,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg1", parameter::string("value"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); } @@ -532,7 +532,7 @@ TEST(RemoteConfigEngineListener, EngineCustomRulesUpdate) auto p = parameter::map(); p.add("arg3", parameter::string("custom rule"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } } @@ -559,7 +559,7 @@ TEST(RemoteConfigEngineListener, EngineRuleDataUpdate) auto p = parameter::map(); p.add("http.client_ip", parameter::string("1.2.3.4"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -572,7 +572,7 @@ TEST(RemoteConfigEngineListener, EngineRuleDataUpdate) auto p = parameter::map(); p.add("http.client_ip", parameter::string("1.2.3.4"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_FALSE(res); } @@ -583,7 +583,7 @@ TEST(RemoteConfigEngineListener, EngineRuleDataUpdate) auto p = parameter::map(); p.add("http.client_ip", parameter::string("1.2.3.4"sv)); - auto res = ctx.publish(std::move(p)); + auto res = ctx.publish(std::move(p), {}); EXPECT_TRUE(res); EXPECT_EQ(res->actions[0].type, dds::action_type::block); EXPECT_EQ(res->triggers.size(), 1); diff --git a/appsec/tests/helper/waf_test.cpp b/appsec/tests/helper/waf_test.cpp index 45d8f2003de..205e8880c59 100644 --- a/appsec/tests/helper/waf_test.cpp +++ b/appsec/tests/helper/waf_test.cpp @@ -126,7 +126,7 @@ TEST(WafTest, RunWithInvalidParam) auto ctx = wi->get_listener(); parameter_view pv; dds::event e; - EXPECT_THROW(ctx->call(pv, e), invalid_object); + EXPECT_THROW(ctx->call(pv, e, {}), invalid_object); } { // Rasp NiceMock submitm{}; @@ -163,7 +163,7 @@ TEST(WafTest, RunWithInvalidParam) parameter_view pv; dds::event e; std::string rasp = "lfi"; - EXPECT_THROW(ctx->call(pv, e, rasp), invalid_object); + EXPECT_THROW(ctx->call(pv, e, {.rasp_rule = rasp}), invalid_object); ctx->submit_metrics(submitm); Mock::VerifyAndClearExpectations(&submitm); } @@ -183,7 +183,7 @@ TEST(WafTest, RunWithTimeout) parameter_view pv(p); dds::event e; - EXPECT_THROW(ctx->call(pv, e), timeout_error); + EXPECT_THROW(ctx->call(pv, e, {}), timeout_error); } { // Rasp NiceMock submitm{}; @@ -204,7 +204,7 @@ TEST(WafTest, RunWithTimeout) parameter_view pv(p); dds::event e; std::string rasp = "lfi"; - EXPECT_THROW(ctx->call(pv, e, rasp), timeout_error); + EXPECT_THROW(ctx->call(pv, e, {.rasp_rule = rasp}), timeout_error); ctx->submit_metrics(submitm); Mock::VerifyAndClearExpectations(&submitm); @@ -224,7 +224,7 @@ TEST(WafTest, ValidRunGood) parameter_view pv(p); dds::event e; - ctx->call(pv, e); // default to rasp=false + ctx->call(pv, e, {}); // default to rasp=false EXPECT_CALL(submitm, submit_span_meta(metrics::event_rules_version, std::string{"1.2.3"})); @@ -253,7 +253,7 @@ TEST(WafTest, ValidRunGood) parameter_view pv(p); dds::event e; std::string rasp = "lfi"; - ctx->call(pv, e, rasp); + ctx->call(pv, e, {.rasp_rule = rasp}); double rasp_duration; double duration; @@ -309,7 +309,7 @@ TEST(WafTest, ValidRunMonitor) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); for (auto &match : e.triggers) { rapidjson::Document doc; @@ -350,7 +350,7 @@ TEST(WafTest, ValidRunMonitorObfuscated) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -384,7 +384,7 @@ TEST(WafTest, ValidRunMonitorObfuscatedFromSettings) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -418,7 +418,7 @@ TEST(WafTest, UpdateRuleData) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); } auto param = json_to_parameter( @@ -441,7 +441,7 @@ TEST(WafTest, UpdateRuleData) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -475,7 +475,7 @@ TEST(WafTest, UpdateInvalid) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); } changeset cs; @@ -506,7 +506,7 @@ TEST(WafTest, SchemasAreAdded) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -545,7 +545,7 @@ TEST(WafTest, FingerprintAreNotAdded) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_CALL(submitm, submit_span_meta_copy_key(MatchesRegex("_dd\\.appsec\\.fp\\..+"), _)) @@ -588,7 +588,7 @@ TEST(WafTest, FingerprintAreAdded) parameter_view pv(p); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_CALL( submitm, submit_span_meta_copy_key("_dd.appsec.fp.http.endpoint", @@ -627,7 +627,7 @@ TEST(WafTest, ActionsAreSentAndParsed) auto ctx = wi->get_listener(); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -666,7 +666,7 @@ TEST(WafTest, ActionsAreSentAndParsed) auto ctx = wi->get_listener(); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -705,7 +705,7 @@ TEST(WafTest, ActionsAreSentAndParsed) auto ctx = wi->get_listener(); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -739,7 +739,7 @@ TEST(WafTest, ActionsAreSentAndParsed) auto ctx = wi->get_listener(); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); EXPECT_EQ(e.triggers.size(), 1); rapidjson::Document doc; @@ -781,12 +781,12 @@ TEST(WafTest, TelemetryIsSent) parameter_view pv(p); dds::event e; std::string rasp = "ssrf"; - ctx->call(pv, e, rasp); + ctx->call(pv, e, {.rasp_rule = rasp}); // Now rasp call without match auto p2 = parameter::map(); parameter_view pv2(p2); - ctx->call(pv2, e, rasp); + ctx->call(pv2, e, {.rasp_rule = rasp}); // Now lfi with match auto p3 = parameter::map(); @@ -795,13 +795,13 @@ TEST(WafTest, TelemetryIsSent) query.add("query", parameter::string("../somefile"sv)); p3.add("server.request.query", std::move(query)); parameter_view pv3(p3); - ctx->call(pv3, e, "lfi"); + ctx->call(pv3, e, {.rasp_rule = "lfi"}); parameter_view pv4; - EXPECT_THROW(ctx->call(pv4, e, "lfi"), invalid_object); + EXPECT_THROW(ctx->call(pv4, e, {.rasp_rule = "lfi"}), invalid_object); parameter_view pv5; - EXPECT_THROW(ctx->call(pv5, e, "lfi"), invalid_object); + EXPECT_THROW(ctx->call(pv5, e, {.rasp_rule = "lfi"}), invalid_object); EXPECT_CALL(submitm, submit_span_meta(metrics::event_rules_version, std::string{"1.2.3"})); @@ -883,7 +883,7 @@ TEST(WafTest, TelemetryTimeoutMetric) parameter_view pv(p); dds::event e; std::string rasp = "lfi"; - EXPECT_THROW(ctx->call(pv, e, rasp), timeout_error); + EXPECT_THROW(ctx->call(pv, e, {.rasp_rule = rasp}), timeout_error); EXPECT_CALL(submitm, submit_span_meta(metrics::event_rules_version, std::string{"1.2.3"})); @@ -952,7 +952,7 @@ TEST(WafTest, TraceAttributesAreSent) auto ctx = wi->get_listener(); dds::event e; - ctx->call(pv, e); + ctx->call(pv, e, {}); ctx->submit_metrics(submitm); Mock::VerifyAndClearExpectations(&submitm); } diff --git a/appsec/tests/integration/build.gradle b/appsec/tests/integration/build.gradle index 10be6deffd5..485f5c8d0c1 100644 --- a/appsec/tests/integration/build.gradle +++ b/appsec/tests/integration/build.gradle @@ -151,23 +151,40 @@ def buildRunInDockerTask = { Map options -> String baseName = options.get('baseName') def version = options.get('version') def variant = options.get('variant') - String imageTag = "${options.get('baseTag', 'php')}-$version-$variant" - String imageName = "$dockerMirror/${repo}${map_tag(imageTag)}" - def pullTask = dockerPullTask(imageTag) + def hasVersionVariant = version && variant + def suffix = hasVersionVariant ? "-${version}-${variant}" : '' + + String imageName + def pullTask + if (options.containsKey('externalImage')) { + // Use external Docker image (e.g., rust:alpine3.23) + imageName = "$dockerMirror/${options['externalImage']}" + pullTask = null // No pull task for external images + } else { + String imageTag = options.get('imageTag') ?: "${options.get('baseTag', 'php')}-$version-$variant" + imageName = "$dockerMirror/${repo}${map_tag(imageTag)}" + pullTask = dockerPullTask(imageTag) + } - def volumes = [:] + def volumes = options.get('volumes', [:]).collectEntries { k, v -> + [k, v + [task: createVolumeTask(k)]] + } def binds = [ ("${projectDir}/../../..".toString()): '/project' ] - if (options.get('needsTracer', true)) { - volumes["php-tracer-${version}-${variant}"] = [ + if (hasVersionVariant && options.get('needsTracer', true)) { + def volName = "php-tracer-${version}-${variant}" + volumes[volName] = [ mountPoint: '/project/tmp', + task: createVolumeTask(volName), ] } - if (options.get('needsAppsec', true)) { - volumes["php-appsec-${version}-${variant}"] = [ + if (hasVersionVariant && options.get('needsAppsec', true)) { + def volName = "php-appsec-${version}-${variant}" + volumes[volName] = [ mountPoint: '/appsec', + task: createVolumeTask(volName), ] } if (options.get('needsBoostCache', true)) { @@ -176,15 +193,18 @@ def buildRunInDockerTask = { Map options -> } else { volumes['php-appsec-boost-cache'] = [ mountPoint: '/root/.boost', + task: createVolumeTask('php-appsec-boost-cache'), ] } } if (options.get('needsCargoCache', true)) { volumes['php-tracer-cargo-cache'] = [ mountPoint: '/root/.cargo/registry', + task: createVolumeTask('php-tracer-cargo-cache'), ] volumes['php-tracer-cargo-cache-git'] = [ mountPoint: '/root/.cargo/git', + task: createVolumeTask('php-tracer-cargo-cache-git'), ] } @@ -199,13 +219,9 @@ def buildRunInDockerTask = { Map options -> binds[composerFile] = '/usr/local/bin/composer' } - volumes.keySet().each { volumeName -> - volumes[volumeName]['task'] = createVolumeTask(volumeName) - } - - def t = tasks.register("$baseName-$version-$variant", Exec) { + def t = tasks.register("$baseName$suffix", Exec) { if (options['description']) { - description = "${options['description']} for PHP $version $variant" + description = hasVersionVariant ? "${options['description']} for PHP $version $variant" : options['description'] def inputsSpec = options.get('inputs', [:]) inputsSpec.get('dirs', []).each { dir -> @@ -231,13 +247,13 @@ def buildRunInDockerTask = { Map options -> if (!options['outputs']) { outputs.upToDateWhen { false } } else { - String volumeName = "${options['outputs']['volume']}-${version}-${variant}" + String volumeName = hasVersionVariant ? "${options['outputs']['volume']}${suffix}" : options['outputs']['volume'] def files = options['outputs']['files'] outputs.upToDateWhen { Process proc = ['docker', 'run', '--rm', '--mount', "type=volume,src=$volumeName,dst=/vol", "$dockerMirror/library/busybox", 'sh', '-c', - "stat -c %Y ${files.collect { "'/vol/$it'" }.join(' ')} | sort -n | head -1"] + "stat -c %Y ${files.collect { "'/vol/$it'" }.join(' ')} 2>/dev/null | sort -n | head -1"] .execute() proc.waitForOrKill(5_000) @@ -261,20 +277,27 @@ def buildRunInDockerTask = { Map options -> def commandLine = [ 'docker', 'run', '--init', '--rm', - '--entrypoint', '/bin/bash', - '--user', uuid, - '-e', 'HOME=/tmp', + '--entrypoint', options.get('entrypoint', '/bin/bash'), ] + if (!options.get('runAsRoot', false)) { + commandLine.addAll(['--user', uuid, '-e', 'HOME=/tmp']) + } binds.each { source, dest -> commandLine.addAll(['--mount', "type=bind,src=${source},dst=${dest}"]) } volumes.each { volumeName, volumeSpec -> commandLine << '--mount' - commandLine << "type=volume,src=${volumeName},dst=${volumeSpec['mountPoint']}" + def mountOpts = "type=volume,src=${volumeName},dst=${volumeSpec['mountPoint']}" + if (volumeSpec.get('readonly', false)) { + mountOpts += ',readonly' + } + commandLine << mountOpts dependsOn volumeSpec['task'] } commandLine << imageName - dependsOn pullTask + if (pullTask) { + dependsOn pullTask + } commandLine.addAll(options['command']) it.commandLine commandLine @@ -282,15 +305,20 @@ def buildRunInDockerTask = { Map options -> if (composerDlTask) { dependsOn composerDlTask } + + options.get('taskDependencies', []).each { dep -> + dependsOn dep + } } } if (options.containsKey('outputs') && options['outputs'].containsKey('volume')) { - String taskName = "cleanVolume-${options['outputs']['volume']}-${version}-${variant}" + String volName = hasVersionVariant ? "${options['outputs']['volume']}${suffix}" : options['outputs']['volume'] + String taskName = "cleanVolume-${volName}" if (!tasks.findByName(taskName)) { def task = tasks.register(taskName, Exec) { - description = "Clean volume ${options['outputs']['volume']} for PHP $version $variant" - commandLine 'docker', 'volume', 'rm', '-f', "${options['outputs']['volume']}-${version}-${variant}" + description = hasVersionVariant ? "Clean volume ${options['outputs']['volume']} for PHP $version $variant" : "Clean volume ${options['outputs']['volume']}" + commandLine 'docker', 'volume', 'rm', '-f', volName } tasks['clean'].dependsOn task } @@ -299,10 +327,10 @@ def buildRunInDockerTask = { Map options -> t } -def buildTracerTask = { String version, String variant -> +def buildTracerTask = { String version, String variant, altBaseTag = null -> buildRunInDockerTask( baseName: 'buildTracer', - baseTag: 'php', + baseTag: altBaseTag ?: 'php', version: version, variant: variant, needsAppsec: false, @@ -329,11 +357,11 @@ def buildTracerTask = { String version, String variant -> ) } -def buildAppSecTask = { String version, String variant -> +def buildAppSecTask = { String version, String variant, altBaseTag = null -> def buildType = variant.contains('debug') ? 'Debug' : 'RelWithDebInfo' buildRunInDockerTask( baseName: 'buildAppsec', - baseTag: 'php', + baseTag: altBaseTag ?: 'php', version: version, variant: variant, needsTracer: false, @@ -407,18 +435,39 @@ def runMainTask = { String phpVersion, String variant -> systemProperty 'PHP_VERSION', phpVersion systemProperty 'VARIANT', variant + if (project.hasProperty('helperBinary')) { + systemProperty 'USE_HELPER_RUST', '1' + systemProperty 'HELPER_BINARY_PATH', project.getProperty('helperBinary') + } else if (project.hasProperty('useHelperRust') || project.hasProperty('useHelperRustCoverage')) { + systemProperty 'USE_HELPER_RUST', '1' + } + if (project.hasProperty('useHelperRustCoverage')) { + systemProperty 'USE_HELPER_RUST_COVERAGE', '1' + } + dependsOn "buildTracer-$phpVersion-$variant" dependsOn "buildAppsec-$phpVersion-$variant" + if (project.hasProperty('helperBinary')) { + // Skip building helper-rust when explicit binary path is provided + } else if (project.hasProperty('useHelperRustCoverage')) { + dependsOn 'buildHelperRustWithCoverage' + } else if (project.hasProperty('useHelperRust')) { + dependsOn 'buildHelperRust' + } } } -testMatrix.each { spec -> +(testMatrix + [['8.5', 'release-musl']]).each { spec -> String phpVersion = spec[0] String variant = spec[1] - buildTracerTask(phpVersion, variant) - buildAppSecTask(phpVersion, variant) - runUnitTestsTask(phpVersion, variant) + def isMusl = variant =~ /\bmusl\b/ + + buildTracerTask(phpVersion, variant, isMusl ? 'nginx-fpm-php' : null) + buildAppSecTask(phpVersion, variant, isMusl ? 'nginx-fpm-php' : null) + if (!isMusl) { + runUnitTestsTask(phpVersion, variant) + } if (project.hasProperty('testClass')) { runMainTask(phpVersion, variant) } @@ -432,6 +481,10 @@ testMatrix.each { spec -> it.useJUnitPlatform { includeEngines('junit-jupiter') excludeEngines('junit-vintage') + + if (isMusl) { + includeTags('musl') + } } it.testClassesDirs = sourceSets.test.output.classesDirs @@ -442,6 +495,15 @@ testMatrix.each { spec -> if (project.hasProperty('XDEBUG')) { it.systemProperty 'XDEBUG', '1' } + if (project.hasProperty('helperBinary')) { + it.systemProperty 'USE_HELPER_RUST', '1' + it.systemProperty 'HELPER_BINARY_PATH', project.getProperty('helperBinary') + } else if (project.hasProperty('useHelperRust') || project.hasProperty('useHelperRustCoverage')) { + it.systemProperty 'USE_HELPER_RUST', '1' + } + if (project.hasProperty('useHelperRustCoverage')) { + it.systemProperty 'USE_HELPER_RUST_COVERAGE', '1' + } it.systemProperty 'DOCKER_MIRROR', dockerMirror if (!project.hasProperty("floatingImageTags")) { @@ -450,8 +512,15 @@ testMatrix.each { spec -> dependsOn "buildTracer-${phpVersion}-${variant}" dependsOn "buildAppsec-${phpVersion}-${variant}" + if (project.hasProperty('helperBinary')) { + // Skip building helper-rust when explicit binary path is provided + } else if (project.hasProperty('useHelperRustCoverage')) { + dependsOn 'buildHelperRustWithCoverage' + } else if (project.hasProperty('useHelperRust')) { + dependsOn 'buildHelperRust' + } - if (version in ['7.0', '7.1']) { + if (phpVersion in ['7.0', '7.1']) { dependsOn downloadComposerOld } else { dependsOn downloadComposer @@ -496,6 +565,269 @@ task saveCaches(type: Exec) { chown \$UUID /build/php-appsec-volume-caches-${arch}.tar.gz""" } +// libddwaf shared library build (for helper-rust) +buildRunInDockerTask( + baseName: 'buildLibddwaf', + imageTag: 'php-deps', + description: 'Build libddwaf shared library', + needsBoostCache: false, + needsCargoCache: false, + inputs: [ + dirs: ['../../third_party/libddwaf'], + ], + outputs: [ + volume: 'php-libddwaf', + files: ['lib/libddwaf.so', 'include/ddwaf.h'], + ], + volumes: [ + 'php-libddwaf': [mountPoint: '/libddwaf-prefix'], + ], + command: [ + '-e', '-c', + """ + git config --global --add safe.directory '*' + + mkdir -p /tmp/libddwaf-build + cd /tmp/libddwaf-build + cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \\ + -DCMAKE_INSTALL_PREFIX=/libddwaf-prefix \\ + -DLIBDDWAF_BUILD_SHARED=ON \\ + -DLIBDDWAF_BUILD_STATIC=OFF \\ + -DLIBDDWAF_TESTING=OFF \\ + /project/appsec/third_party/libddwaf + make -j install + touch /libddwaf-prefix/lib/libddwaf.so /libddwaf-prefix/include/ddwaf.h + """ + ] +) + +// Shared configuration for helper-rust tasks +def helperRustInputs = [ + dirs: ['../../helper-rust/src', '../../third_party/libddwaf-rust'], + files: [ + '../../helper-rust/Cargo.toml', + '../../helper-rust/Cargo.lock', + '../../helper-rust/build.rs', + '../../helper-rust/glibc_compat.c', + '../../helper-rust/coverage_init.c', + '../../helper-rust/outline_atomics.c', + '../../helper-rust/.cargo/config.toml', + ], +] + +def helperRustEnvSetup = ''' +git config --global --add safe.directory '*' +export PATH="/root/.cargo/bin:$PATH" +export RUSTUP_HOME=/root/.rustup +export CARGO_HOME=/root/.cargo +cd /project/appsec/helper-rust +export CARGO_TARGET_DIR=/helper-rust-build/cargo-target +''' + +buildRunInDockerTask( + baseName: 'buildHelperRust', + imageTag: 'nginx-fpm-php-8.5-release-musl', + entrypoint: '/bin/sh', + description: 'Build helper-rust with musl for universal compatibility', + needsBoostCache: false, + needsCargoCache: false, + inputs: helperRustInputs, + outputs: [ + volume: 'php-helper-rust', + files: ['libddappsec-helper.so'], + ], + volumes: [ + 'php-helper-rust': [mountPoint: '/helper-rust-build'], + 'php-tracer-cargo-cache': [mountPoint: '/usr/local/cargo/registry'], + 'php-tracer-cargo-cache-git': [mountPoint: '/usr/local/cargo/git'], + ], + command: [ + '-e', '-c', + ''' + export CARGO_TARGET_DIR=/helper-rust-build/cargo-target + RUST_TARGET=$(uname -m)-unknown-linux-musl + + git config --global --add safe.directory '*' + + cd /project/appsec/helper-rust + + # Build using nightly toolchain with unstable features + # -Z build-std: Rebuild std library for musl + # -Z build-std-features=llvm-libunwind: Use LLVM libunwind instead of libgcc_s + cargo +nightly-"$RUST_TARGET" build \\ + -Zhost-config \\ + -Ztarget-applies-to-host \\ + --target "$RUST_TARGET" + + # Remove musl libc dependency using patchelf (makes binary work on both musl and glibc) + BINARY_PATH="/helper-rust-build/cargo-target/$RUST_TARGET/debug/libddappsec_helper_rust.so" + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + patchelf --remove-needed libc.musl-x86_64.so.1 "$BINARY_PATH" 2>/dev/null || true + elif [ "$ARCH" = "aarch64" ]; then + patchelf --remove-needed libc.musl-aarch64.so.1 "$BINARY_PATH" 2>/dev/null || true + fi + + cp "$BINARY_PATH" /helper-rust-build/libddappsec-helper.so'''.stripLeading() + ] +) + +buildRunInDockerTask( + baseName: 'testHelperRust', + imageTag: 'php-deps', + description: 'Test and lint helper-rust', + needsBoostCache: false, + inputs: helperRustInputs, + volumes: [ + 'php-helper-rust': [mountPoint: '/helper-rust-build'], + ], + command: [ + '-e', '-c', + """ + ${helperRustEnvSetup} + + echo '=== Checking formatting ===' + cargo fmt --check + + echo '=== Building helper-rust ===' + cargo build --release + + echo '=== Running cargo test ===' + cargo test --release + + echo '=== Build and test successful ===' + """ + ] +) + +buildRunInDockerTask( + baseName: 'coverageHelperRust', + imageTag: 'php-deps', + description: 'Generate coverage for helper-rust unit tests', + needsBoostCache: false, + inputs: helperRustInputs, + outputs: [ + volume: 'php-helper-rust-coverage', + files: ['coverage-unit.lcov'], + ], + volumes: [ + 'php-helper-rust-coverage': [mountPoint: '/helper-rust-build'], + ], + command: [ + '-e', '-c', + """ + ${helperRustEnvSetup} + + echo '=== Installing cargo-llvm-cov ===' + rustup component add llvm-tools-preview + ARCH=\$(uname -m) + if [ "\$ARCH" = "aarch64" ]; then + LLVM_COV_ARCH="aarch64-unknown-linux-musl" + else + LLVM_COV_ARCH="x86_64-unknown-linux-musl" + fi + curl -LsSf "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.6.23/cargo-llvm-cov-\${LLVM_COV_ARCH}.tar.gz" | tar xzf - -C /root/.cargo/bin + + echo '=== Running cargo test with coverage ===' + cargo llvm-cov test --release --lcov --output-path /helper-rust-build/coverage-unit.lcov + + echo '=== Coverage data generated ===' + """ + ] +) + +// Build helper-rust with coverage instrumentation for integration tests +buildRunInDockerTask( + baseName: 'buildHelperRustWithCoverage', + imageTag: 'php-deps', + description: 'Build helper-rust with coverage instrumentation', + needsBoostCache: false, + inputs: helperRustInputs, + outputs: [ + volume: 'php-helper-rust-coverage', + files: ['libddappsec-helper.so'], + ], + volumes: [ + 'php-helper-rust-coverage': [mountPoint: '/helper-rust-build'], + ], + command: [ + '-e', '-c', + """ + ${helperRustEnvSetup} + + # Determine target triple + ARCH=\$(uname -m) + if [ "\$ARCH" = "aarch64" ]; then + RUST_TARGET="aarch64-unknown-linux-gnu" + else + RUST_TARGET="x86_64-unknown-linux-gnu" + fi + + RUST_TARGET_UPPER=\$(echo "\${RUST_TARGET//-/_}" | tr '[:lower:]' '[:upper:]') + export CARGO_TARGET_\${RUST_TARGET_UPPER}_RUSTFLAGS="-C instrument-coverage" + export LLVM_PROFILE_FILE='/helper-rust-build/coverage/default-%m-%p.profraw' + + mkdir -p /helper-rust-build/coverage + # Make coverage dir world-writable so sidecar (running as www-data) can write profraw files + chmod 777 /helper-rust-build/coverage + + cargo build --release --target \$RUST_TARGET --features coverage + cp /helper-rust-build/cargo-target/\$RUST_TARGET/release/libddappsec_helper_rust.so /helper-rust-build/libddappsec-helper.so + """ + ] +) + +// Generate coverage report from integration test profraw files +buildRunInDockerTask( + baseName: 'generateHelperRustIntegrationCoverage', + imageTag: 'php-deps', + description: 'Generate coverage report from helper-rust integration tests', + needsBoostCache: false, + inputs: [dirs: []], + outputs: [ + volume: 'php-helper-rust-coverage', + files: ['coverage-integration.lcov'], + ], + volumes: [ + 'php-helper-rust-coverage': [mountPoint: '/helper-rust-build'], + ], + taskDependencies: [], + command: [ + '-e', '-c', + """ + export PATH="/root/.cargo/bin:\$PATH" + export RUSTUP_HOME=/root/.rustup + export CARGO_HOME=/root/.cargo + + cd /helper-rust-build/coverage + if ls *.profraw 1>/dev/null 2>&1; then + echo 'Found profraw files:' + ls -la *.profraw + + # Detect the architecture-specific toolchain path + ARCH=\$(uname -m) + if [ "\$ARCH" = "aarch64" ]; then + RUST_TARGET="aarch64-unknown-linux-gnu" + else + RUST_TARGET="x86_64-unknown-linux-gnu" + fi + LLVM_TOOLS="/root/.rustup/toolchains/1.84.1-\${RUST_TARGET}/lib/rustlib/\${RUST_TARGET}/bin" + + \${LLVM_TOOLS}/llvm-profdata merge -sparse *.profraw -o merged.profdata + \${LLVM_TOOLS}/llvm-cov export \\ + /helper-rust-build/libddappsec-helper.so \\ + -format=lcov \\ + -instr-profile=merged.profdata \\ + > /helper-rust-build/coverage-integration.lcov + echo 'Coverage report generated' + else + echo 'No profraw files found - coverage may not have been collected' + exit 1 + fi + """ + ] +) + if (hasProperty('buildScan')) { buildScan { termsOfServiceUrl = 'https://gradle.com/terms-of-service' diff --git a/appsec/tests/integration/gradle/images.gradle b/appsec/tests/integration/gradle/images.gradle index b405df03bd6..b32e23eee34 100644 --- a/appsec/tests/integration/gradle/images.gradle +++ b/appsec/tests/integration/gradle/images.gradle @@ -37,7 +37,12 @@ def phpVersions = [ '8.5': '8.5.0', ] -def arch = System.getProperty('os.arch') +def nativeArch = System.getProperty('os.arch') +def arch = project.hasProperty('dockerArch') ? project.property('dockerArch') : nativeArch +def archToSuffix = { a -> a == 'arm64' ? 'aarch64' : a } +def archSuffix = archToSuffix(arch) +def isCrossBuilding = arch != nativeArch +def dockerPlatform = arch == 'arm64' ? 'linux/arm64' : 'linux/amd64' def imageUpToDate = { inputs, String image -> return { @@ -55,6 +60,7 @@ def imageIsNewerThan = { image1, image2 -> final List dockerBuildCommand = [ 'docker', 'buildx', 'build', '--progress=plain', '--load', + *(isCrossBuilding ? ['--platform', dockerPlatform] : []), ] tasks.register('buildToolchain', Exec) { @@ -201,18 +207,32 @@ tasks.register('buildFrankenPHP-8.4-release-zts', Exec) { 'src/docker/frankenphp', '-f', 'src/docker/frankenphp/Dockerfile') } +tasks.register('buildNginxFpm-8.5-release-musl', Exec) { + String image = "$repo:nginx-fpm-php-8.5-release-musl" + description = "Build the musl-nginx image for Alpine testing" + inputs.file file('src/docker/nginx-fpm-musl/Dockerfile') + inputs.file file('src/docker/nginx-fpm-musl/nginx.conf') + outputs.upToDateWhen imageUpToDate(inputs, image) + + commandLine(*dockerBuildCommand, *dockerMirrorArgs, + '-t', image, + '-f', 'src/docker/nginx-fpm-musl/Dockerfile', + 'src/docker/nginx-fpm-musl') +} + task buildAll { dependsOn 'buildAllPhp', 'buildAllApache2Mod', 'buildAllApache2Fpm', 'buildAllNginxFpm', - 'buildFrankenPHP-8.4-release-zts' + 'buildFrankenPHP-8.4-release-zts', + 'buildNginxFpm-8.5-release-musl' } def buildPushTask = { String tag, requirement -> tasks.register("pushImage-${tag}", Exec) { String image = "$repo:$tag" - String pushedImage = image + "-$arch" + String pushedImage = image + "-$archSuffix" description = "Push image $image" doFirst { @@ -243,6 +263,7 @@ def allPushTasks = [ buildPushTask("nginx-fpm-php-${spec[0]}-${spec[1]}", "buildNginxFpm-${spec[0]}-${spec[1]}") }, buildPushTask("frankenphp-8.4-release-zts", 'buildFrankenPHP-8.4-release-zts'), + buildPushTask("nginx-fpm-php-8.5-release-musl", 'buildNginxFpm-8.5-release-musl'), ] tasks.register('pushAll') { dependsOn allPushTasks @@ -279,7 +300,8 @@ def allMultiArchTasks = [ *testMatrix.collect { spec -> buildMultiArchTask("nginx-fpm-php-${spec[0]}-${spec[1]}") }, - buildMultiArchTask("frankenphp-8.4-release-zts") + buildMultiArchTask("frankenphp-8.4-release-zts"), + buildMultiArchTask("nginx-fpm-php-8.5-release-musl"), ] tasks.register('pushMultiArch') { dependsOn allMultiArchTasks diff --git a/appsec/tests/integration/gradle/tag_mappings.gradle b/appsec/tests/integration/gradle/tag_mappings.gradle index b21812e2d05..e66859f946b 100644 --- a/appsec/tests/integration/gradle/tag_mappings.gradle +++ b/appsec/tests/integration/gradle/tag_mappings.gradle @@ -82,7 +82,7 @@ ext.tag_mappings = [ 'nginx-fpm-php-7.2-release-zts': 'sha256:40b1171928cb1a8be69c3f571d6bdd57ab22f8ef6cf88fe34e3c68971fdd4ad9', 'nginx-fpm-php-7.1-release-zts': 'sha256:eb5e55128f9e91e93a9d4cd918777eceb163a88343db1d2679b73e0463769ce4', 'nginx-fpm-php-7.0-release-zts': 'sha256:e2ae077387dc8af90ea4722df2558cbc90a76f04c01b65ed064151493e56ef81', - 'php-deps': 'sha256:ba5c6d1cfeb82a7552ba57d24cca1c905c72b170e71f730c752f7b8bed35d82c', + 'php-deps': 'sha256:b9601fa1937c745403f65a0f4919acf3883b812b843c0b7cd8e0ae817f217a52', 'nginx-fpm-php-7.1-release': 'sha256:e7e42bb9410aee145bae2827a4d242e17dcd692ec8a4486bc5712788c4b3005c', 'nginx-fpm-php-7.2-debug': 'sha256:20e284b3936a45547a60d7090d40aa1e57caea85fbc8ada666a97a959a863a0b', 'apache2-mod-php-7.4-debug': 'sha256:2b76e1949737fd52ebea0227099b5a6821ea675008ea9091d8d8274ab8c47873', @@ -136,5 +136,6 @@ ext.tag_mappings = [ 'apache2-mod-php-8.5-debug': 'sha256:1ce0570b1e8fba34db0e4ec0f18134fb30c1d50ba795b6f42f0cbe3f19c7a68f', 'nginx-fpm-php-8.5-release-zts': 'sha256:527f2ba273685ff7e3f72a74076fd6d99cbc87d51fb3485fef4dc80a0cfddeee', 'nginx-fpm-php-8.5-release': 'sha256:fc2eea1ed06dfcf8102bdbfb2596f9a32bbe6493ca83d53370e1438289caa6aa', - 'nginx-fpm-php-8.5-debug': 'sha256:915d0155897e4f42cf1867b93edbb928afcc1c5b3bd3eccdb2d56ac50510cd82' + 'nginx-fpm-php-8.5-debug': 'sha256:915d0155897e4f42cf1867b93edbb928afcc1c5b3bd3eccdb2d56ac50510cd82', + 'nginx-fpm-php-8.5-release-musl': 'sha256:73857002765b217e1076adc04f9ebc88505b0fa77fc48e2ad3005594b2f39bc9', ] diff --git a/appsec/tests/integration/src/docker/nginx-fpm-musl/Dockerfile b/appsec/tests/integration/src/docker/nginx-fpm-musl/Dockerfile new file mode 100644 index 00000000000..572184d33df --- /dev/null +++ b/appsec/tests/integration/src/docker/nginx-fpm-musl/Dockerfile @@ -0,0 +1,41 @@ +FROM php:8.5-fpm-alpine + +# doubles as php/toolchain image for musl + +RUN apk add --no-cache \ + autoconf bash bison clang clang-dev cmake curl curl-dev \ + g++ gcc gdb git libc-dev linux-headers llvm-libunwind-static make \ + musl-dev nginx oniguruma-dev openssl-dev patchelf pkgconf re2c \ + libxml2-dev libzip-dev xz-static zlib-dev \ + vim + + +RUN adduser -D -u 1000 linux_user + +ENV CARGO_HOME=/usr/local/cargo \ + RUSTUP_HOME=/usr/local/rustup \ + PATH="/usr/local/cargo/bin:${PATH}" \ + LIBCLANG_PATH=/usr/lib + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \ + sh -s -- -y --default-toolchain 1.84.1 --no-modify-path && \ + chmod -R 777 /usr/local/cargo /usr/local/rustup && \ + ln -sf /usr/local/cargo/bin/{cargo,rustc} /usr/local/bin/ && \ + rustup toolchain install nightly-$(uname -m)-unknown-linux-musl && \ + rustup component add rust-src --toolchain nightly-$(uname -m)-unknown-linux-musl + +RUN mkdir -p /run/nginx && \ + mkdir -p /var/www/public && \ + chown -R linux_user:linux_user /var/www && \ + mkdir -p /etc/php && \ + touch /etc/php/php.ini && \ + chmod 666 /etc/php/php.ini +COPY nginx.conf /etc/nginx/http.d/default.conf +COPY php-fpm.conf /usr/local/etc/php-fpm.d/www.conf + +COPY start.sh /start.sh +RUN chmod +x /start.sh + +WORKDIR /project + +CMD ["/start.sh"] diff --git a/appsec/tests/integration/src/docker/nginx-fpm-musl/nginx.conf b/appsec/tests/integration/src/docker/nginx-fpm-musl/nginx.conf new file mode 100644 index 00000000000..90297e79c24 --- /dev/null +++ b/appsec/tests/integration/src/docker/nginx-fpm-musl/nginx.conf @@ -0,0 +1,15 @@ +server { + listen 80 default_server; + root /var/www/public; + index index.php; + + location / { + try_files $uri $uri/ =404; + } + + location ~ \.php$ { + fastcgi_pass 127.0.0.1:9000; + fastcgi_index index.php; + include fastcgi.conf; + } +} diff --git a/appsec/tests/integration/src/docker/nginx-fpm-musl/php-fpm.conf b/appsec/tests/integration/src/docker/nginx-fpm-musl/php-fpm.conf new file mode 100644 index 00000000000..2907507f716 --- /dev/null +++ b/appsec/tests/integration/src/docker/nginx-fpm-musl/php-fpm.conf @@ -0,0 +1,20 @@ +[global] +daemonize = no +error_log = /proc/self/fd/2 + +[www] +user = linux_user +group = linux_user +listen = 127.0.0.1:9000 +pm = dynamic +pm.max_children = 5 +pm.start_servers = 2 +pm.min_spare_servers = 1 +pm.max_spare_servers = 3 +access.log = /proc/self/fd/2 +clear_env = no +catch_workers_output = yes + +;Default is 10000 so lets give it 1 more +;in order to test pool envs +env[DD_APPSEC_WAF_TIMEOUT] = "10001" diff --git a/appsec/tests/integration/src/docker/nginx-fpm-musl/start.sh b/appsec/tests/integration/src/docker/nginx-fpm-musl/start.sh new file mode 100644 index 00000000000..c0791b3a3c7 --- /dev/null +++ b/appsec/tests/integration/src/docker/nginx-fpm-musl/start.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -ex + +# Create log directories and files (matches nginx-fpm/entrypoint.sh) +mkdir -p /tmp/logs +LOGS_PHP=( + /tmp/logs/appsec.log + /tmp/logs/helper.log + /tmp/logs/php_error.log + /tmp/logs/php_fpm_error.log + /tmp/logs/sidecar.log +) +touch "${LOGS_PHP[@]}" +chown linux_user "${LOGS_PHP[@]}" + +# Enable extensions (writes ddtrace/ddappsec to php.ini) +enable_extensions.sh + +# Start PHP-FPM in the background with custom php.ini location +php-fpm -c /etc/php & + +# Start nginx in the foreground +exec nginx -g 'daemon off;' diff --git a/appsec/tests/integration/src/docker/php/Dockerfile-php-deps b/appsec/tests/integration/src/docker/php/Dockerfile-php-deps index 5d0283723e5..ed62102399a 100644 --- a/appsec/tests/integration/src/docker/php/Dockerfile-php-deps +++ b/appsec/tests/integration/src/docker/php/Dockerfile-php-deps @@ -16,11 +16,14 @@ RUN apt-get update && apt-get install -y \ procps \ python3-dev \ vim \ + libclang-dev \ && rm -rf /var/lib/apt/lists/* ADD build_dev_php.sh /build/php/ RUN USER=root /build/php/build_dev_php.sh deps -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.84.1 -y +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.84.1 -y \ + && /root/.cargo/bin/rustup component add llvm-tools-preview \ + && chmod -R a+rX /root /root/.cargo /root/.rustup ENV PATH="/root/.cargo/bin:${PATH}" diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/TelemetryHelpers.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/TelemetryHelpers.groovy index e90748cb956..87cffe0ef3e 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/TelemetryHelpers.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/TelemetryHelpers.groovy @@ -1,6 +1,7 @@ package com.datadog.appsec.php import groovy.transform.Canonical +import groovy.transform.ToString import java.net.http.HttpRequest import java.net.http.HttpResponse import com.datadog.appsec.php.docker.AppSecContainer @@ -26,6 +27,7 @@ class TelemetryHelpers { } } + @ToString static class AppEndpoints { static names = ['app-endpoints'] List endpoints @@ -59,6 +61,7 @@ class TelemetryHelpers { } } + @ToString static class Metric { String namespace String name @@ -93,6 +96,7 @@ class TelemetryHelpers { } @Canonical + @ToString static class Log { String level String message diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy index 7e985539c56..028fa5251fd 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/AppSecContainer.groovy @@ -232,7 +232,85 @@ class AppSecContainer> extends GenericContain 5_000 - (Math.max(0, System.currentTimeMillis() - start))) } } + /** + * Apply remote config with raw byte content instead of JSON-encoded maps. + * This allows testing with malformed/corrupted config data. + * + * @param target The RC target + * @param files Map of config paths to raw byte arrays + * @return Supplier that waits for RC request confirmation + */ + Supplier applyRemoteConfigRaw(Target target, Map files) { + Map encodedFiles = files.findAll { it.value != null } + long newVersion = Instant.now().epochSecond + def rcr = new RemoteConfigResponse() + rcr.clientConfigs = files.keySet() as List + rcr.targetFiles = encodedFiles.collect { + new RemoteConfigResponse.TargetFile( + path: it.key, + raw: new String( + Base64.encoder.encode(it.value), + StandardCharsets.ISO_8859_1) + ) + } + rcr.targets = new RemoteConfigResponse.Targets( + signatures: [], + targetsSigned: new RemoteConfigResponse.Targets.TargetsSigned( + type: 'root', + custom: new RemoteConfigResponse.Targets.TargetsSigned.TargetsCustom( + opaqueBackendState: 'ABCDEF' + ), + specVersion: '1.0.0', + expires: Instant.parse('2030-01-01T00:00:00Z'), + version: newVersion, + targets: encodedFiles.collectEntries { + [ + it.key, + new RemoteConfigResponse.Targets.ConfigTarget( + hashes: [sha256: RemoteConfigResponse.sha256(it.value).toString(16).padLeft(64, '0')], + length: it.value.size(), + custom: new RemoteConfigResponse.Targets.ConfigTarget.ConfigTargetCustom( + version: newVersion + ) + ) + ] + } + ), + ) + + setNextRCResponse(target, rcr) + + long start = System.currentTimeMillis() + return { -> waitForRCVersion(target, newVersion, + 5_000 - (Math.max(0, System.currentTimeMillis() - start))) } + } + + void flushProfilingData() { + if (!System.getProperty('USE_HELPER_RUST_COVERAGE')) { + return + } + + try { + ExecResult res = execInContainer('/bin/bash', '-c', + ''' + pid=$(pgrep -f '[d]atadog-ipc-helper' || true) + if [ -n "$pid" ]; then + echo "Sending SIGUSR1 to helper/sidecar process to flush coverage: $pid" >&2 + kill -USR1 $pid + # Give it a moment to flush coverage + sleep 0.5 + fi + ''') + if (res.exitCode != 0) { + log.warn("Failed to cleanup helper: ${res.stderr}") + } + } catch (Exception e) { + log.warn("Exception during helper cleanup: ${e.message}") + } + } + void close() { + flushProfilingData() copyLogs() super.close() } @@ -372,6 +450,28 @@ class AppSecContainer> extends GenericContain '/usr/local/bin/enable_extensions.sh', BindMode.READ_ONLY) addVolumeMount("php-appsec-$phpVersion-$phpVariant", '/appsec') addVolumeMount("php-tracer-$phpVersion-$phpVariant", '/project/tmp') + if (System.getProperty('USE_HELPER_RUST')) { + String helperBinaryPath = System.getProperty('HELPER_BINARY_PATH') + if (helperBinaryPath) { + // Bind-mount explicit helper binary directly to the expected path + File helperFile = new File(helperBinaryPath) + if (!helperFile.isAbsolute()) { + helperFile = new File(System.getProperty('user.dir'), helperBinaryPath) + } + withFileSystemBind(helperFile.absolutePath, + '/helper-rust/libddappsec-helper.so', BindMode.READ_ONLY) + } else { + // libddwaf is statically linked into the helper-rust binary + String helperVolume = System.getProperty('USE_HELPER_RUST_COVERAGE') ? + 'php-helper-rust-coverage' : 'php-helper-rust' + addVolumeMount(helperVolume, '/helper-rust') + if (System.getProperty('USE_HELPER_RUST_COVERAGE')) { + // Enable LLVM coverage profiling for the helper binary + withEnv 'LLVM_PROFILE_FILE', '/helper-rust/coverage/default-%m-%p.profraw' + } + } + withEnv 'USE_HELPER_RUST', '1' + } String fullWorkVolume = "php-workvol-$workVolume-$phpVersion-$phpVariant" diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/FailOnUnmatchedTracesExtension.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/FailOnUnmatchedTracesExtension.groovy index 5701e520fb3..fd527cb9e44 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/FailOnUnmatchedTracesExtension.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/docker/FailOnUnmatchedTracesExtension.groovy @@ -18,6 +18,7 @@ class FailOnUnmatchedTracesExtension implements AfterEachCallback { throw new RuntimeException( '@FailOnUnmatchedTraces can only be applied to AppSecContainer fields') } + if (context.executionException.present) { container.clearTraces() } else { diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/TelemetryHandler.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/TelemetryHandler.groovy index 97fc0ab9128..80d4737993c 100644 --- a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/TelemetryHandler.groovy +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/mock_agent/TelemetryHandler.groovy @@ -55,7 +55,9 @@ class TelemetryHandler implements Handler { List drain(long timeoutInMs) { synchronized (capturedTelemetryMessages) { if (!savedError && capturedTelemetryMessages.isEmpty()) { - capturedTelemetryMessages.wait(timeoutInMs) + if (timeoutInMs != 0) { + capturedTelemetryMessages.wait(timeoutInMs) + } } if (savedError) { def e = savedError diff --git a/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/test/ClearTelemetryExtension.groovy b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/test/ClearTelemetryExtension.groovy new file mode 100644 index 00000000000..1cbf94326a6 --- /dev/null +++ b/appsec/tests/integration/src/main/groovy/com/datadog/appsec/php/test/ClearTelemetryExtension.groovy @@ -0,0 +1,28 @@ +package com.datadog.appsec.php.test + +import com.datadog.appsec.php.docker.AppSecContainer +import groovy.util.logging.Slf4j +import org.junit.jupiter.api.extension.BeforeEachCallback +import org.junit.jupiter.api.extension.ExtensionContext + +@Slf4j +class ClearTelemetryExtension implements BeforeEachCallback { + @Override + void beforeEach(ExtensionContext context) throws Exception { + Class testClass = context.requiredTestClass + try { + def containerField = testClass.getDeclaredField('CONTAINER') + containerField.accessible = true + + AppSecContainer container = containerField.get(null) // Assuming the field is static + def tel = container?.drainTelemetry(0) + if (tel) { + log.info("Cleared ${tel.size()} telemetry messages before '${context.displayName}'") + } + } catch (NoSuchFieldException ignored) { + // No action needed if the field does not exist + } catch (Exception e) { + throw new RuntimeException("Error stopping the container", e) + } + } +} diff --git a/appsec/tests/integration/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/appsec/tests/integration/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension index 28a76b597a2..fcab63662b0 100644 --- a/appsec/tests/integration/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension +++ b/appsec/tests/integration/src/main/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -1,2 +1,3 @@ +com.datadog.appsec.php.test.ClearTelemetryExtension com.datadog.appsec.php.test.NotifyTestLifecycle com.datadog.appsec.php.test.StopContainerExtension diff --git a/appsec/tests/integration/src/test/bin/enable_extensions.sh b/appsec/tests/integration/src/test/bin/enable_extensions.sh index 6ce13d2ee62..5470e7c0f0e 100755 --- a/appsec/tests/integration/src/test/bin/enable_extensions.sh +++ b/appsec/tests/integration/src/test/bin/enable_extensions.sh @@ -17,18 +17,25 @@ if [[ -f /project/tmp/build_extension/modules/ddtrace.so ]]; then } >> /etc/php/php.ini fi +HELPER_PATH=/appsec/libddappsec-helper.so +if [[ -n $USE_HELPER_RUST ]]; then + echo "Using Rust helper" >&2 + HELPER_PATH=/helper-rust/libddappsec-helper.so +fi + if [[ -f /appsec/ddappsec.so && -d /project ]]; then echo "Enabling ddappsec" >&2 { echo extension=/appsec/ddappsec.so echo datadog.appsec.enabled=true - echo datadog.appsec.helper_path=/appsec/libddappsec-helper.so + echo datadog.appsec.helper_path=$HELPER_PATH echo datadog.appsec.helper_log_file=/tmp/logs/helper.log echo datadog.appsec.helper_log_level=debug echo datadog.appsec.rules=/etc/recommended.json echo datadog.appsec.log_file=/tmp/logs/appsec.log echo datadog.appsec.log_level=debug echo datadog.appsec.rasp_enabled=1 + echo datadog.appsec.testing_invalid_command=1 } >> /etc/php/php.ini fi diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy index 32e69271dbc..9be5159b761 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/Apache2FpmTests.groovy @@ -79,6 +79,7 @@ class Apache2FpmTests implements CommonTests, SamplingTestsInFpm, EndpointFallba } void setRateLimit(String limit) { + flushProfilingData() def res = container.execInContainer( 'bash', '-c', """kill -9 `pgrep php-fpm`; @@ -88,6 +89,7 @@ class Apache2FpmTests implements CommonTests, SamplingTestsInFpm, EndpointFallba } private void resetFpm() { + flushProfilingData() container.execInContainer( 'bash', '-c', '''kill -9 `pgrep php-fpm`; diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/CommonTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/CommonTests.groovy index aa0b1144435..cfc0e2cd1a7 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/CommonTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/CommonTests.groovy @@ -10,6 +10,8 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.containers.Container +import groovy.json.JsonSlurper +import groovy.json.JsonOutput import java.net.http.HttpRequest import java.net.http.HttpResponse @@ -26,6 +28,47 @@ trait CommonTests { getClass().CONTAINER } + void cleanupHelperBeforeRestart() { + // Clean up helper before restarting services to ensure coverage is flushed + container.flushProfilingData() + } + + /** + * Normalizes AppSec JSON to handle differences between libddwaf versions. + * - Converts array indices in key_path to strings (libddwaf 1.x format) + * - Removes on_match field (added in libddwaf 2.x) + */ + String normalizeAppsecJson(String json) { + def slurper = new JsonSlurper() + def parsed = slurper.parseText(json) + + // Recursively process the structure + def normalized = normalizeAppsecObject(parsed) + + JsonOutput.toJson(normalized) + } + + private Object normalizeAppsecObject(Object obj) { + if (obj instanceof Map) { + def result = [:] + obj.each { key, value -> + // Skip on_match field (libddwaf 2.x addition) + if (key != 'on_match') { + // Convert array indices to strings in key_path + if (key == 'key_path' && value instanceof List) { + result[key] = value.collect { it instanceof Number ? it.toString() : it } + } else { + result[key] = normalizeAppsecObject(value) + } + } + } + return result + } else if (obj instanceof List) { + return obj.collect { normalizeAppsecObject(it) } + } + return obj + } + @Test void 'user tracking'() { def trace = container.traceFromRequest('/user_id.php') { HttpResponse resp -> @@ -628,7 +671,9 @@ trait CommonTests { } ] }''' - assertThat appsecJson, matchesJson(expJson, false, true) + // Normalize to handle libddwaf version differences (array index types, on_match field) + def normalizedAppsecJson = normalizeAppsecJson(appsecJson) + assertThat normalizedAppsecJson, matchesJson(expJson, false, true) } @Test @@ -674,7 +719,9 @@ trait CommonTests { } ] }''' - assertThat appsecJson, matchesJson(expJson, false, true) + // Normalize to handle libddwaf version differences (array index types, on_match field) + def normalizedAppsecJson = normalizeAppsecJson(appsecJson) + assertThat normalizedAppsecJson, matchesJson(expJson, false, true) } diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy index d69c28d15a9..6c051a9ebd9 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/EndpointFallbackSamplingTests.groovy @@ -1,7 +1,6 @@ package com.datadog.appsec.php.integration import com.datadog.appsec.php.docker.AppSecContainer -import com.datadog.appsec.php.model.Trace import org.junit.jupiter.api.Test import java.net.http.HttpResponse @@ -111,6 +110,7 @@ trait EndpointFallbackSamplingTests extends SamplingTestsInFpm { } void disableEndpointRenaming() { + flushProfilingData() def res = container.execInContainer( 'bash', '-c', '''kill -9 `pgrep php-fpm`; diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/NginxFpmTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/NginxFpmTests.groovy index 25ec33cdd4e..8b22701d5d6 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/NginxFpmTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/NginxFpmTests.groovy @@ -4,6 +4,7 @@ import com.datadog.appsec.php.docker.AppSecContainer import com.datadog.appsec.php.docker.FailOnUnmatchedTraces import com.datadog.appsec.php.docker.InspectContainerHelper import groovy.util.logging.Slf4j +import org.junit.jupiter.api.Tag import org.junit.jupiter.api.Test import org.junit.jupiter.api.condition.DisabledIf import org.testcontainers.junit.jupiter.Container @@ -17,6 +18,7 @@ import static com.datadog.appsec.php.integration.TestParams.getVariant @Testcontainers @Slf4j @DisabledIf('isZts') +@Tag("musl") class NginxFpmTests implements CommonTests { static boolean zts = variant.contains('zts') diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RemoteConfigTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RemoteConfigTests.groovy index f807748a83a..3d20fbb646a 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RemoteConfigTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/RemoteConfigTests.groovy @@ -42,6 +42,7 @@ class RemoteConfigTests { @BeforeAll static void beforeAll() { + CONTAINER.flushProfilingData() ExecResult res = CONTAINER.execInContainer( 'bash', '-c', '''sed -e '/appsec.enabled/d' -e '/appsec.rules=/d' /etc/php/php.ini > /etc/php/php-rc.ini; diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/SamplingTestsInFpm.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/SamplingTestsInFpm.groovy index e95cd4ca16d..7fbea7b5ec3 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/SamplingTestsInFpm.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/SamplingTestsInFpm.groovy @@ -102,6 +102,7 @@ trait SamplingTestsInFpm { } void setSamplingPeriod(String period) { + flushProfilingData() def res = container.execInContainer( 'bash', '-c', """kill -9 `pgrep php-fpm`; @@ -111,9 +112,14 @@ trait SamplingTestsInFpm { } private void resetFpm() { + flushProfilingData() container.execInContainer( 'bash', '-c', '''kill -9 `pgrep php-fpm`; php-fpm -y /etc/php-fpm.conf -c /etc/php/php.ini''') } + + void flushProfilingData() { + container.flushProfilingData() + } } \ No newline at end of file diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/TelemetryTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/TelemetryTests.groovy index 3911041cf5a..2d1857a734c 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/TelemetryTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/TelemetryTests.groovy @@ -7,6 +7,7 @@ import com.datadog.appsec.php.mock_agent.rem_cfg.RemoteConfigRequest import com.datadog.appsec.php.mock_agent.rem_cfg.Target import com.datadog.appsec.php.model.Trace import groovy.util.logging.Slf4j +import org.junit.jupiter.api.Assumptions import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.MethodOrderer import org.junit.jupiter.api.Order @@ -18,6 +19,7 @@ import org.testcontainers.junit.jupiter.Testcontainers import java.net.http.HttpRequest import java.net.http.HttpResponse +import java.nio.charset.StandardCharsets import java.util.function.Supplier import static com.datadog.appsec.php.integration.TestParams.getPhpVersion @@ -42,10 +44,17 @@ class TelemetryTests { phpVersion: phpVersion, phpVariant: variant, www: 'base', - ) + ) { + @Override + void configure() { + super.configure() + withEnv('RUST_LIB_BACKTRACE', '1') + } + } @BeforeAll static void beforeAll() { + CONTAINER.flushProfilingData() org.testcontainers.containers.Container.ExecResult res = CONTAINER.execInContainer( 'bash', '-c', '''sed -e '/appsec.enabled/d' -e '/appsec.rules=/d' /etc/php/php.ini > /etc/php/php-rc.ini; @@ -68,7 +77,7 @@ class TelemetryTests { ] ]) - // first request to start helper + // first request to start helper - triggers WAF init with legacy span metrics // Generally won't be covered by appsec because it doesn't receive RC data in time // for the response to config_sync Trace trace = CONTAINER.traceFromRequest('/hello.php') { HttpResponse resp -> @@ -76,6 +85,12 @@ class TelemetryTests { } assert trace.traceId != null + // Check legacy span metrics from WAF init are present + def initSpan = trace[0] + assert initSpan.metrics.'_dd.appsec.event_rules.loaded' > 0 + assert initSpan.metrics.'_dd.appsec.event_rules.error_count' == 0.0d + assert initSpan.meta.'_dd.appsec.event_rules.errors' == '{}' + RemoteConfigRequest rcReq = requestSup.get() assert rcReq != null, 'No RC request received' @@ -84,6 +99,10 @@ class TelemetryTests { assert resp.statusCode() == 200 } + // Check legacy span meta is present (metrics only on init) + def span = trace[0] + assert span.meta.'_dd.appsec.event_rules.version' =~ /\d+\.\d+\.\d+/ + // now do an attack HttpRequest req = CONTAINER.buildReq('/hello.php') .header('User-Agent', 'Arachni/v1').GET().build() @@ -102,8 +121,10 @@ class TelemetryTests { TelemetryHelpers.waitForMetrics(CONTAINER, 30) { List messages -> def allSeries = messages.collectMany { it.series } wafInit = allSeries.find { it.name == 'waf.init' } - wafReq1 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == 2 } - wafReq2 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == 3 } + // Rust helper has +1 tag (helper_runtime), C++ doesn't + def useRust = System.getProperty('USE_HELPER_RUST') != null + wafReq1 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == (useRust ? 3 : 2) } + wafReq2 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == (useRust ? 4 : 3) } connSuccess = allSeries.find { it.name == 'helper.connection_success' } workerCount = allSeries.find { it.name == 'helper.service_worker_count' } @@ -114,7 +135,6 @@ class TelemetryTests { assert wafInit.namespace == 'appsec' assert wafInit.points[0][1] == 1.0 assert 'success:true' in wafInit.tags - assert wafInit.tags.size() == 3 assert wafInit.type == 'count' assert wafInit.interval == 10 @@ -138,6 +158,21 @@ class TelemetryTests { assert workerCount != null assert workerCount.namespace == 'appsec' assert workerCount.points[0][1] >= 1.0 + + // Check helper_runtime tag: only Rust helper should have it + if (System.getProperty('USE_HELPER_RUST') != null) { + assert 'helper_runtime:rust' in wafInit.tags + assert 'helper_runtime:rust' in wafReq1.tags + assert 'helper_runtime:rust' in wafReq2.tags + assert 'helper_runtime:rust' in workerCount.tags + // connSuccess is from extension, not helper, so it doesn't have helper_runtime tag + } else { + // C++ helper should NOT have the helper_runtime tag in telemetry + assert !wafInit.tags.any { it.startsWith('helper_runtime:') } + assert !wafReq1.tags.any { it.startsWith('helper_runtime:') } + assert !wafReq2.tags.any { it.startsWith('helper_runtime:') } + assert !workerCount.tags.any { it.startsWith('helper_runtime:') } + } } @Test @@ -210,18 +245,24 @@ class TelemetryTests { ] ]) + TelemetryHelpers.Metric wafUpdates def messages = TelemetryHelpers.waitForMetrics(CONTAINER, 30) { List messages -> - def allSeries = messages - .collectMany { it.series } - .findAll { - it.name == 'waf.config_errors' - } + def allSeries = messages.collectMany { it.series } + def configErrors = allSeries.findAll { it.name == 'waf.config_errors' } + wafUpdates = allSeries.find { it.name == 'waf.updates' } - allSeries.size() >= 4 + configErrors.size() >= 4 && wafUpdates } assert requestSup.get() != null + // Make a request after RC is confirmed applied, check span has new version + def trace = CONTAINER.traceFromRequest('/hello.php') { HttpResponse resp -> + assert resp.statusCode() == 200 + } + def span = trace[0] + assert span.meta.'_dd.appsec.event_rules.version' == '1.1.1' + def series = messages .collectMany { it.series } .findAll { @@ -254,6 +295,24 @@ class TelemetryTests { assert customRules.points[0][1] == 1.0d assert rules.points[0][1] == 1.0d assert data.points[0][1] == 1.0d + + assert wafUpdates != null + assert wafUpdates.namespace == 'appsec' + assert wafUpdates.points[0][1] >= 1.0d + assert 'success:true' in wafUpdates.tags + assert 'event_rules_version:1.1.1' in wafUpdates.tags + assert wafUpdates.tags.find { it.startsWith('waf_version:') } != null + assert wafUpdates.type == 'count' + + // Check helper_runtime tag: only Rust helper should have it + if (System.getProperty('USE_HELPER_RUST') != null) { + assert 'helper_runtime:rust' in wafUpdates.tags + series.each { assert 'helper_runtime:rust' in it.tags } + } else { + // C++ helper should NOT have the helper_runtime tag in telemetry + assert !wafUpdates.tags.any { it.startsWith('helper_runtime:') } + series.each { assert !it.tags.any { tag -> tag.startsWith('helper_runtime:') } } + } } @Test @@ -307,29 +366,23 @@ class TelemetryTests { assert messages.any { it.level == 'ERROR' && it.message == "bad cast, expected 'array', obtained 'string'" && - it.parsedTags == [ - log_type: 'rc::asm_data::diagnostic', - appsec_config_key: 'rules_data', - rc_config_id: 'bad_config', - ] + it.parsedTags.log_type == 'rc::asm_data::diagnostic' && + it.parsedTags.appsec_config_key == 'rules_data' && + it.parsedTags.rc_config_id == 'bad_config' } assert messages.any { it.level == 'ERROR' && it.message == "{\"missing key 'conditions'\":[\"bad_rule\"]}" && - it.parsedTags == [ - log_type: 'rc::asm_dd::diagnostic', - appsec_config_key: 'rules', - rc_config_id: 'bad_rule', - ] + it.parsedTags.log_type == 'rc::asm_dd::diagnostic' && + it.parsedTags.appsec_config_key == 'rules' && + it.parsedTags.rc_config_id == 'bad_rule' } assert messages.any { it.level == 'WARN' && it.message == "{\"unknown operator: 'unknown_operator'\":[\"bad_condition_rule\"]}" && - it.parsedTags == [ - log_type: 'rc::asm_dd::diagnostic', - appsec_config_key: 'rules', - rc_config_id: 'warning_rule', - ] + it.parsedTags.log_type == 'rc::asm_dd::diagnostic' && + it.parsedTags.appsec_config_key == 'rules' && + it.parsedTags.rc_config_id == 'warning_rule' } } @@ -403,9 +456,10 @@ class TelemetryTests { TelemetryHelpers.Metric lfiTimeout TelemetryHelpers.Metric ssrfTimeout + def useRust = System.getProperty('USE_HELPER_RUST') != null TelemetryHelpers.waitForMetrics(CONTAINER, 30) { List messages -> def allSeries = messages.collectMany { it.series } - wafReq1 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == 2 } + wafReq1 = allSeries.find { it.name == 'waf.requests' && it.tags.size() == (useRust ? 3 : 2) } lfiEval = allSeries.find{ it.name == 'rasp.rule.eval' && 'rule_type:lfi' in it.tags} lfiMatch = allSeries.find{ it.name == 'rasp.rule.match' && 'rule_type:lfi' in it.tags} lfiTimeout = allSeries.find{ it.name == 'rasp.timeout' && 'rule_type:lfi' in it.tags} @@ -458,6 +512,15 @@ class TelemetryTests { assert ssrfTimeout.points[0][1] == 0.0 assert ssrfTimeout.type == 'count' assert ssrfTimeout.tags.find { it.startsWith('waf_version:') } != null + + // Check helper_runtime tag: only Rust helper should have it + def raspMetrics = [wafReq1, lfiEval, lfiMatch, lfiTimeout, ssrfEval, ssrfMatch, ssrfTimeout] + if (System.getProperty('USE_HELPER_RUST') != null) { + raspMetrics.each { assert 'helper_runtime:rust' in it.tags } + } else { + // C++ helper should NOT have the helper_runtime tag in telemetry + raspMetrics.each { assert !it.tags.any { tag -> tag.startsWith('helper_runtime:') } } + } } /** @@ -565,5 +628,131 @@ class TelemetryTests { assert wafReqTruncated.tags.find { it.startsWith('event_rules_version:') } != null assert wafReqTruncated.tags.find { it.startsWith('waf_version:') } != null assert wafReqTruncated.type == 'count' + + // Check helper_runtime tag: only Rust helper should have it + if (System.getProperty('USE_HELPER_RUST') != null) { + assert 'helper_runtime:rust' in wafReqTruncated.tags + } else { + // C++ helper should NOT have the helper_runtime tag in telemetry + assert !wafReqTruncated.tags.any { it.startsWith('helper_runtime:') } + } + } + + /** + * This test verifies that helper-rust errors are properly sent to telemetry + * with backtraces. It sends an invalid message to the helper which triggers + * an error with backtrace. + * + * This test only runs when USE_HELPER_RUST is set (Rust helper implementation). + */ + @Test + @Order(7) + void 'helper error telemetry includes backtrace'() { + Assumptions.assumeTrue(System.getProperty('USE_HELPER_RUST') != null) + + Supplier requestSup = CONTAINER.applyRemoteConfig(RC_TARGET, [ + 'datadog/2/ASM_FEATURES/asm_features_activation/config': [ + asm: [enabled: true] + ] + ]) + + // first request to start helper and establish connection + Trace trace = CONTAINER.traceFromRequest('/hello.php') { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assert trace.traceId != null + + RemoteConfigRequest rcReq = requestSup.get() + assert rcReq != null, 'No RC request received' + + // request covered by Appsec to ensure we have an active connection + trace = CONTAINER.traceFromRequest('/hello.php') { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assert trace.traceId != null + + // send invalid message to trigger error with backtrace + CONTAINER.traceFromRequest('/send_invalid_msg.php') { + HttpResponse resp -> assert resp.statusCode() == 200 + } + + def messages = TelemetryHelpers.waitForLogs(CONTAINER, 30) { List logs -> + def relevantLogs = logs.collectMany { it.logs.findAll { it.tags.contains('log_type:helper::logged_error') } } + !relevantLogs.empty + }.collectMany { it.logs } + + def errorLog = messages.find { it.tags?.contains('log_type:helper::logged_error') } + + assert errorLog != null : "Expected to find a log with log_type:helper::client_error. " + + "All logs: ${messages}" + assert errorLog.level == 'ERROR' : "Expected ERROR level, got ${errorLog.level}" + assert errorLog.message?.contains('unknown command') || errorLog.message?.contains('invalid_command') : + "Expected error message about unknown/invalid command, got: ${errorLog.message}" + + // back trace + assert errorLog.stack_trace != null : "Expected stack_trace to be present" + assert errorLog.stack_trace.length() > 0 : "Expected stack_trace to be non-empty" + + // check that the backtrace contains some typical Rust stack frame indicators + def hasStackFrameIndicators = errorLog.stack_trace.contains('::') || + errorLog.stack_trace.contains('.rs:') || + errorLog.stack_trace =~ /at \d+:\d+/ || + errorLog.stack_trace =~ /\d+: / || + errorLog.stack_trace =~ /:\d+$/ + + assert hasStackFrameIndicators : + "Expected backtrace with stack frame indicators (::, .rs:, line numbers), got: ${errorLog.stack_trace}" + + // This test only runs for Rust helper, so verify helper_runtime:rust tag is present in logs + assert errorLog.tags?.contains('helper_runtime:rust') : + "Expected helper_runtime:rust tag in log tags, got: ${errorLog.tags}" + + CONTAINER.clearTraces() + } + + @Test + @Order(8) + void 'telemetry log for malformed RC config JSON'() { + def enableSup = CONTAINER.applyRemoteConfig(RC_TARGET, [ + 'datadog/2/ASM_FEATURES/asm_features_activation/config': [ + asm: [enabled: true] + ] + ]) + + def trace = CONTAINER.traceFromRequest('/hello.php') { HttpResponse resp -> + assert resp.statusCode() == 200 + } + assert trace.traceId != null + assert enableSup.get() != null + + def malformedJson = 'this is not valid JSON {][ at all'.getBytes(StandardCharsets.UTF_8) + + def requestSup = CONTAINER.applyRemoteConfigRaw(RC_TARGET, [ + 'datadog/2/ASM_DD/malformed_config/config': malformedJson + ]) + + // Wait for telemetry logs with rc::asm_dd::exception + def messages = TelemetryHelpers.waitForLogs(CONTAINER, 30) { List logs -> + def relevantLogs = logs.collectMany { + it.logs.findAll { it.tags?.contains('log_type:rc::asm_dd::exception') } + } + !relevantLogs.empty + }.collectMany { it.logs } + + assert requestSup.get() != null + + def exceptionLog = messages.find { + it.tags?.contains('log_type:rc::asm_dd::exception') && + it.tags?.contains('rc_config_id:malformed_config') + } + + assert exceptionLog != null : "Expected to find rc::asm_dd::exception log. " + + "All logs: ${messages.collect { [identifier: it.identifier, tags: it.tags, message: it.message] }}" + assert exceptionLog.level == 'ERROR' : "Expected ERROR level, got ${exceptionLog.level}" + assert exceptionLog.message?.contains('malformed') || + exceptionLog.message?.contains('parse') || + exceptionLog.message?.contains('JSON') || + exceptionLog.message?.contains('Failed to apply config') : + "Expected error message about parse/JSON failure, got: ${exceptionLog.message}" } } diff --git a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/WorkerStrategyTests.groovy b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/WorkerStrategyTests.groovy index 54f32bc50da..dcbe607e7cf 100644 --- a/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/WorkerStrategyTests.groovy +++ b/appsec/tests/integration/src/test/groovy/com/datadog/appsec/php/integration/WorkerStrategyTests.groovy @@ -3,6 +3,10 @@ package com.datadog.appsec.php.integration import com.datadog.appsec.php.docker.AppSecContainer import com.datadog.appsec.php.model.Mapper import com.datadog.appsec.php.model.Span +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.ObjectNode import org.junit.jupiter.api.Test import java.net.http.HttpRequest @@ -14,6 +18,8 @@ import static org.hamcrest.MatcherAssert.assertThat import static org.junit.jupiter.api.Assumptions.assumeTrue trait WorkerStrategyTests { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper() + AppSecContainer getContainer() { getClass().CONTAINER } @@ -21,6 +27,40 @@ trait WorkerStrategyTests { abstract boolean getCanBlockOnResponse() abstract CharSequence getComponent() + /** + * Normalizes key_path arrays in the actual JSON to use integers for array indices. + * libddwaf v1.x returned strings like "3", v2.x returns integers like 3. + * This normalizes to the v2.x format (integers) for consistent comparison. + */ + private static String normalizeKeyPathInJson(String json) { + JsonNode node = JSON_MAPPER.readTree(json) + normalizeKeyPathNode(node) + JSON_MAPPER.writeValueAsString(node) + } + + private static void normalizeKeyPathNode(JsonNode node) { + if (node.isObject()) { + ObjectNode objNode = (ObjectNode) node + node.fields().each { entry -> + if (entry.key == 'key_path' && entry.value.isArray()) { + ArrayNode normalizedArray = JSON_MAPPER.createArrayNode() + entry.value.each { element -> + if (element.isTextual() && element.asText().matches(/\d+/)) { + normalizedArray.add(element.asText().toInteger()) + } else { + normalizedArray.add(element) + } + } + objNode.set('key_path', normalizedArray) + } else { + normalizeKeyPathNode(entry.value) + } + } + } else if (node.isArray()) { + node.each { element -> normalizeKeyPathNode(element) } + } + } + @Test void 'produces two traces for two requests'() { def trace1 = container.traceFromRequest('/') { HttpResponse it -> @@ -132,7 +172,7 @@ trait WorkerStrategyTests { Span span = trace.first() - def appsecJson = span.meta."_dd.appsec.json" + def appsecJson = normalizeKeyPathInJson(span.meta."_dd.appsec.json") def expJson = '''{ "triggers" : [ { @@ -156,7 +196,7 @@ trait WorkerStrategyTests { ], "key_path" : [ "message", - "3" + 3 ], "value" : "poison" } @@ -180,7 +220,7 @@ trait WorkerStrategyTests { Span span = trace.first() - def appsecJson = span.meta."_dd.appsec.json" + def appsecJson = normalizeKeyPathInJson(span.meta."_dd.appsec.json") def expJson = '''{ "triggers" : [ { @@ -207,7 +247,7 @@ trait WorkerStrategyTests { ], "key_path" : [ "message", - "3" + 3 ], "value" : "block_this" } @@ -229,7 +269,7 @@ trait WorkerStrategyTests { Span span = trace.first() - def appsecJson = span.meta."_dd.appsec.json" + def appsecJson = normalizeKeyPathInJson(span.meta."_dd.appsec.json") def expJson = '''{ "triggers" : [ { @@ -253,7 +293,7 @@ trait WorkerStrategyTests { ], "key_path" : [ "note", - "2" + 2 ], "value" : "\\n poison\\n" } @@ -281,7 +321,7 @@ trait WorkerStrategyTests { assert span.meta['http.request.headers.content-type'] == 'application/json' assert span.meta['http.request.headers.content-length'] == '45' - def appsecJson = span.meta."_dd.appsec.json" + def appsecJson = normalizeKeyPathInJson(span.meta."_dd.appsec.json") def expJson = '''{ "triggers" : [ { @@ -305,7 +345,7 @@ trait WorkerStrategyTests { ], "key_path" : [ "message", - "3" + 3 ], "value" : "poison" } @@ -332,7 +372,7 @@ trait WorkerStrategyTests { Span span = trace.first() - def appsecJson = span.meta."_dd.appsec.json" + def appsecJson = normalizeKeyPathInJson(span.meta."_dd.appsec.json") def expJson = '''{ "triggers" : [ { @@ -356,7 +396,7 @@ trait WorkerStrategyTests { ], "key_path" : [ "note", - "2" + 2 ], "value" : "poison" } diff --git a/appsec/tests/integration/src/test/www/base/public/send_invalid_msg.php b/appsec/tests/integration/src/test/www/base/public/send_invalid_msg.php new file mode 100644 index 00000000000..ca77a53cbd9 --- /dev/null +++ b/appsec/tests/integration/src/test/www/base/public/send_invalid_msg.php @@ -0,0 +1,12 @@ +