diff --git a/.github/workflows/tero-release-please.yaml b/.github/workflows/tero-release-please.yaml new file mode 100644 index 0000000000000..4d7d19262e860 --- /dev/null +++ b/.github/workflows/tero-release-please.yaml @@ -0,0 +1,44 @@ +--- +# Release Please — Tero fork +# +# Watches `master` for commits using the Conventional Commits format and +# opens a "release PR" that bumps the prerelease version (.X-tero.N → .X-tero.N+1) +# and updates `Cargo.toml`, `CHANGELOG.md`, and `.release-please-manifest.json`. +# Merging that PR creates the `v` tag, which triggers +# `tero-release.yaml` to publish artifacts. +# +# This is the Tero-fork equivalent of `tero-collector-distro`'s +# `release-please.yaml`. Do not touch upstream Vector's `release.yml`. + +name: Tero release please + +on: + push: + branches: + - master + +permissions: + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + release-please: + runs-on: ubuntu-24.04 + # Don't fire in the upstream `vectordotdev/vector` repo if this fork's + # `master` is ever pushed back upstream by accident. + if: github.repository_owner == 'usetero' + steps: + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 + with: + # GITHUB_TOKEN works, but PRs it creates cannot trigger other + # workflows. If you want `tero-release.yaml` to fire when the + # release PR is merged (and the tag is pushed automatically), swap + # this for a PAT or GitHub App token. Today the tag push DOES + # trigger workflows because release-please pushes it directly. + token: ${{ secrets.GITHUB_TOKEN }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json diff --git a/.github/workflows/tero-release.yaml b/.github/workflows/tero-release.yaml new file mode 100644 index 0000000000000..8e26e7150a1e7 --- /dev/null +++ b/.github/workflows/tero-release.yaml @@ -0,0 +1,361 @@ +--- +# Tero release pipeline. +# +# Triggers on `v*` tags created by `tero-release-please.yaml` (or `workflow_dispatch` +# for a smoke test). Builds the Vector binary natively on amd64 + arm64 +# Linux runners, produces a tarball per arch, then bakes both into a +# multi-arch Docker image at `ghcr.io/${REPO}:VERSION`, and finally attaches +# the tarballs to the GitHub Release that release-please already created. +# +# This pipeline is intentionally narrow — no .deb / .rpm / .msi / macOS / +# Windows / musl / armv7. The upstream `release.yml` / `publish.yml` +# pipelines that produce those artifacts remain untouched but should be +# disabled in repository Settings → Actions → Workflows for this fork +# (see distribution/docker/tero/README.md). + +name: Tero release + +on: + push: + tags: + - "v*" + workflow_dispatch: + inputs: + version: + description: "Version to release (without v prefix). Use for dry runs." + required: false + default: "0.0.0-tero.dryrun" + dry_run: + description: "Build but do not push images or attach release assets." + required: false + default: false + type: boolean + +permissions: + contents: write + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + CARGO_TERM_COLOR: always + RUSTFLAGS: "-C strip=symbols" + +jobs: + prepare: + name: Prepare release + runs-on: ubuntu-24.04 + if: github.repository_owner == 'usetero' + outputs: + version: ${{ steps.version.outputs.version }} + tag: ${{ steps.version.outputs.tag }} + steps: + - name: Determine version + id: version + env: + INPUT_VERSION: ${{ inputs.version }} + REF: ${{ github.ref }} + EVENT_NAME: ${{ github.event_name }} + run: | + if [ "${EVENT_NAME}" = "workflow_dispatch" ]; then + VERSION="${INPUT_VERSION}" + else + VERSION="${REF#refs/tags/v}" + fi + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + echo "tag=v${VERSION}" >> "${GITHUB_OUTPUT}" + echo "Resolved version: ${VERSION}" + + build-binary: + name: Build binary (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: prepare + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + triple: x86_64-unknown-linux-gnu + - arch: arm64 + runner: ubuntu-24.04-arm + triple: aarch64-unknown-linux-gnu + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install build dependencies + # Vectorscan (via `vectorscan-rs-sys`) needs cmake + a C++ toolchain + # + ragel + libpcre2. Everything else (libsasl2, libssl, krb5) is + # required by Vector's default `target-*` feature set. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libboost-dev \ + libclang-dev \ + libpcre2-dev \ + libprotobuf-dev \ + libsasl2-dev \ + libssl-dev \ + libxml2-dev \ + pkg-config \ + protobuf-compiler \ + ragel \ + unzip + # `target-*-linux-gnu` pulls in `vendored` which builds krb5/gssapi + # from source; install its build-time prereqs too. + sudo apt-get install -y --no-install-recommends \ + bison \ + comerr-dev \ + libkrb5-dev + + - name: Install Rust toolchain + # rust-toolchain.toml pins the channel — the action respects it. + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + rustflags: "" + + - name: Cache cargo registry & build + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: tero-release-${{ matrix.triple }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + tero-release-${{ matrix.triple }}- + + - name: Build Vector + env: + TRIPLE: ${{ matrix.triple }} + run: | + cargo build \ + --release \ + --target "${TRIPLE}" \ + --no-default-features \ + --features "target-${TRIPLE},transforms-policy" + + - name: Stage binary artifacts + env: + VERSION: ${{ needs.prepare.outputs.version }} + TRIPLE: ${{ matrix.triple }} + run: | + ARCHIVE_DIR="vector-${VERSION}-${TRIPLE}" + mkdir -p "${ARCHIVE_DIR}" + cp "target/${TRIPLE}/release/vector" "${ARCHIVE_DIR}/vector" + cp LICENSE "${ARCHIVE_DIR}/LICENSE" + cp README.md "${ARCHIVE_DIR}/README.md" + chmod +x "${ARCHIVE_DIR}/vector" + tar -czf "${ARCHIVE_DIR}.tar.gz" "${ARCHIVE_DIR}" + sha256sum "${ARCHIVE_DIR}.tar.gz" > "${ARCHIVE_DIR}.tar.gz.sha256" + ls -lh "${ARCHIVE_DIR}.tar.gz" + + - name: Verify binary + env: + TRIPLE: ${{ matrix.triple }} + run: "./target/${TRIPLE}/release/vector --version" + + - name: Upload tarball artifact + uses: actions/upload-artifact@v4 + with: + name: vector-binary-${{ matrix.arch }} + path: | + vector-*.tar.gz + vector-*.tar.gz.sha256 + if-no-files-found: error + retention-days: 1 + + - name: Upload bare binary for Docker stage + uses: actions/upload-artifact@v4 + with: + name: vector-bin-${{ matrix.arch }} + path: target/${{ matrix.triple }}/release/vector + if-no-files-found: error + retention-days: 1 + + build-docker: + name: Build Docker image (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: [prepare, build-binary] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + platform: linux/amd64 + - arch: arm64 + runner: ubuntu-24.04-arm + platform: linux/arm64 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download binary for this arch + uses: actions/download-artifact@v4 + with: + name: vector-bin-${{ matrix.arch }} + path: ${{ matrix.arch }}/ + + - name: Assemble build context + run: | + chmod +x "${{ matrix.arch }}/vector" + ls -la "${{ matrix.arch }}/" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + if: ${{ !inputs.dry_run }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: distribution/docker/tero/Dockerfile + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ !inputs.dry_run }} + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + + - name: Export digest + if: ${{ !inputs.dry_run }} + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p /tmp/digests + touch "/tmp/digests/${DIGEST#sha256:}" + + - name: Upload digest + if: ${{ !inputs.dry_run }} + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-docker-manifest: + name: Merge Docker manifests + runs-on: ubuntu-24.04 + needs: [prepare, build-docker] + if: ${{ !inputs.dry_run }} + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ needs.prepare.outputs.version }} + type=raw,value=latest,enable=${{ github.event_name == 'push' }} + type=sha + + - name: Create multi-arch manifest and push + working-directory: /tmp/digests + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + run: | + # shellcheck disable=SC2046 + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "${DOCKER_METADATA_OUTPUT_JSON}") \ + $(printf "${IMAGE}@sha256:%s " *) + + - name: Inspect image + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.prepare.outputs.version }} + run: docker buildx imagetools inspect "${IMAGE}" + + upload-release-assets: + name: Attach binaries to GitHub Release + runs-on: ubuntu-24.04 + needs: [prepare, build-binary, merge-docker-manifest] + if: ${{ !inputs.dry_run && github.event_name == 'push' }} + permissions: + contents: write + steps: + - name: Download binary tarballs + uses: actions/download-artifact@v4 + with: + path: /tmp/assets + pattern: vector-binary-* + merge-multiple: true + + - name: List assets + run: ls -la /tmp/assets + + - name: Upload to GitHub Release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ needs.prepare.outputs.tag }} + # `gh release upload` requires the release to already exist. release-please + # creates it at tag time; this job runs immediately after, so the release + # should be there. `--clobber` lets us re-run safely. + run: | + cd /tmp/assets + gh release upload "${TAG}" \ + --repo "${{ github.repository }}" \ + --clobber \ + vector-*.tar.gz \ + vector-*.tar.gz.sha256 + + summary: + name: Release summary + runs-on: ubuntu-24.04 + needs: [prepare, build-binary, build-docker, merge-docker-manifest, upload-release-assets] + if: always() + steps: + - name: Summarize + env: + VERSION: ${{ needs.prepare.outputs.version }} + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + { + echo "## Tero Vector release" + echo + echo "Version: \`${VERSION}\`" + echo + if [ "${DRY_RUN}" = "true" ]; then + echo "**Dry run.** Artifacts built; nothing pushed." + else + echo "Docker image:" + echo + echo " docker pull ${IMAGE}:${VERSION}" + echo + echo "Binary tarballs:" + echo + echo "* \`vector-${VERSION}-x86_64-unknown-linux-gnu.tar.gz\`" + echo "* \`vector-${VERSION}-aarch64-unknown-linux-gnu.tar.gz\`" + fi + } >> "${GITHUB_STEP_SUMMARY}" diff --git a/.gitignore b/.gitignore index a713eeb297f9a..24a8428749602 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ cargo-timing*.html **/*.rs.bk target +target-test tests/data/wasm/*/target bench_output.txt rust-analyzer.toml diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000000000..cbd43ad2679a3 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.56.0-tero.0" +} diff --git a/Cargo.lock b/Cargo.lock index f8841d4be18e1..547fef2781ae2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4365,6 +4365,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -5944,6 +5954,17 @@ dependencies = [ "snafu 0.7.5", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + [[package]] name = "inotify" version = "0.11.0" @@ -5984,6 +6005,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -7500,6 +7530,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" +[[package]] +name = "notify" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +dependencies = [ + "bitflags 2.10.0", + "filetime", + "fsevent-sys", + "inotify 0.10.2", + "kqueue", + "libc", + "log", + "mio", + "notify-types 1.0.1", + "walkdir", + "windows-sys 0.52.0", +] + [[package]] name = "notify" version = "8.2.0" @@ -7508,16 +7557,25 @@ checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ "bitflags 2.10.0", "fsevent-sys", - "inotify", + "inotify 0.11.0", "kqueue", "libc", "log", "mio", - "notify-types", + "notify-types 2.0.0", "walkdir", "windows-sys 0.60.2", ] +[[package]] +name = "notify-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585d3cb5e12e01aed9e8a1f70d5c6b5e86fe2a6e48fc8cd0b3e0b8df6f6eb174" +dependencies = [ + "instant", +] + [[package]] name = "notify-types" version = "2.0.0" @@ -8147,6 +8205,28 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5a797f0e07bdf071d15742978fc3128ec6c22891c31a3a931513263904c982a" +[[package]] +name = "pbjson" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" +dependencies = [ + "heck 0.5.0", + "itertools 0.13.0", + "prost 0.13.5", + "prost-types 0.13.5", +] + [[package]] name = "pbkdf2" version = "0.11.0" @@ -8390,6 +8470,28 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "policy-rs" +version = "1.6.0" +source = "git+https://github.com/usetero/policy-rs?rev=fc94f17312aa51019276c97f7fb57abf20089bf5#fc94f17312aa51019276c97f7fb57abf20089bf5" +dependencies = [ + "fnv", + "notify 7.0.0", + "pbjson", + "pbjson-build", + "prost 0.13.5", + "prost-build 0.13.5", + "rand 0.8.5", + "regex", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", + "tonic 0.12.3", + "tonic-build 0.12.3", + "vectorscan-rs-sys", +] + [[package]] name = "polling" version = "3.7.4" @@ -11577,6 +11679,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tcp-stream" version = "0.34.2" @@ -12251,6 +12364,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tonic-build" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +dependencies = [ + "prettyplease", + "proc-macro2 1.0.106", + "prost-build 0.13.5", + "prost-types 0.13.5", + "quote 1.0.45", + "syn 2.0.117", +] + [[package]] name = "tonic-build" version = "0.13.1" @@ -13127,7 +13254,7 @@ dependencies = [ "nix 0.31.2", "nkeys", "nom 8.0.0", - "notify", + "notify 8.2.0", "opendal", "openssl", "openssl-probe", @@ -13137,6 +13264,7 @@ dependencies = [ "pastey", "percent-encoding", "pin-project", + "policy-rs", "postgres-openssl", "procfs", "proptest", @@ -13647,6 +13775,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "vectorscan-rs-sys" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473f4b86c4c9e42140712b9790079bca554f6341c8ce19f0f1a3ad3b9509dbd5" +dependencies = [ + "cmake", + "flate2", + "tar", +] + [[package]] name = "version_check" version = "0.9.4" @@ -14802,6 +14941,16 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix 1.1.4", +] + [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 3290b7192da7d..052e34131533d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,6 +338,9 @@ prost-types = { workspace = true, optional = true } # Databricks Zerobus databricks-zerobus-ingest-sdk = { git = "https://github.com/databricks/zerobus-sdk-rs", rev = "273efe09130a7554e6b34d20b95df50289e58524", optional = true } +# Tero Policy (policy-rs) — used by the `policy` transform. +policy-rs = { git = "https://github.com/usetero/policy-rs", rev = "fc94f17312aa51019276c97f7fb57abf20089bf5", optional = true } + # GCP goauth = { version = "0.16.0", optional = true } smpl_jwt = { version = "0.8.0", default-features = false, optional = true } @@ -825,6 +828,7 @@ transforms-window = [] transforms-log_to_metric = [] transforms-lua = ["dep:mlua", "vector-lib/lua"] transforms-metric_to_log = [] +transforms-policy = ["dep:policy-rs", "policy-rs/http", "policy-rs/grpc"] transforms-reduce = ["transforms-impl-reduce"] transforms-remap = [] transforms-route = [] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index fa77ca0482eb8..c4a7f61cb5bdf 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -280,6 +280,7 @@ fancy-regex,https://github.com/fancy-regex/fancy-regex,MIT,"Raph Levien ff,https://github.com/zkcrypto/ff,MIT OR Apache-2.0,"Sean Bowe , Jack Grigg " fiat-crypto,https://github.com/mit-plv/fiat-crypto,MIT OR Apache-2.0 OR BSD-1-Clause,Fiat Crypto library authors +filetime,https://github.com/alexcrichton/filetime,MIT OR Apache-2.0,Alex Crichton finl_unicode,https://github.com/dahosek/finl_unicode,MIT OR Apache-2.0,The finl_unicode Authors flatbuffers,https://github.com/google/flatbuffers,Apache-2.0,"Robert Winslow , FlatBuffers Maintainers" flate2,https://github.com/rust-lang/flate2-rs,MIT OR Apache-2.0,"Alex Crichton , Josh Triplett " @@ -388,6 +389,7 @@ inotify,https://github.com/hannobraun/inotify,ISC,"Hanno Braun inout,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers instability,https://github.com/ratatui-org/instability,MIT,"Stephen M. Coakley , Joshka" +instant,https://github.com/sebcrozet/instant,BSD-3-Clause,sebcrozet integer-encoding,https://github.com/dermesser/integer-encoding-rs,MIT,Lewin Bormann inventory,https://github.com/dtolnay/inventory,MIT OR Apache-2.0,David Tolnay ipconfig,https://github.com/liranringel/ipconfig,MIT OR Apache-2.0,Liran Ringel @@ -548,6 +550,7 @@ parquet,https://github.com/apache/arrow-rs,Apache-2.0,Apache Arrow paste,https://github.com/dtolnay/paste,MIT OR Apache-2.0,David Tolnay pastey,https://github.com/as1100k/pastey,MIT OR Apache-2.0,"Aditya Kumar , David Tolnay " +pbjson,https://github.com/influxdata/pbjson,MIT,Raphael Taylor-Davies pbkdf2,https://github.com/RustCrypto/password-hashes/tree/master/pbkdf2,MIT OR Apache-2.0,RustCrypto Developers peeking_take_while,https://github.com/fitzgen/peeking_take_while,MIT OR Apache-2.0,Nick Fitzgerald pem,https://github.com/jcreekmore/pem-rs,MIT,Jonathan Creekmore @@ -567,6 +570,7 @@ piper,https://github.com/notgull/piper,MIT OR Apache-2.0,"Stjepan Glavina +policy-rs,https://github.com/usetero/policy-rs,Apache-2.0,The policy-rs Authors polling,https://github.com/smol-rs/polling,Apache-2.0 OR MIT,"Stjepan Glavina , John Nunley " poly1305,https://github.com/RustCrypto/universal-hashes,Apache-2.0 OR MIT,RustCrypto Developers portable-atomic,https://github.com/taiki-e/portable-atomic,Apache-2.0 OR MIT,The portable-atomic Authors @@ -858,6 +862,7 @@ utf8parse,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Dylan DPC, Hunar Roop Kahlon" uuid-simd,https://github.com/Nugine/simd,MIT,The uuid-simd Authors valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors +vectorscan-rs-sys,https://github.com/bradlarsen/vectorscan-rs,Apache-2.0 OR MIT,Brad Larsen vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " diff --git a/changelog.d/policy_transform.feature.md b/changelog.d/policy_transform.feature.md new file mode 100644 index 0000000000000..4c0df1ef9a11d --- /dev/null +++ b/changelog.d/policy_transform.feature.md @@ -0,0 +1,3 @@ +Added a new `policy` transform that evaluates log events against a JSON policy file using the [`policy-rs`](https://github.com/usetero/policy-rs) engine. The transform can keep, drop, sample, and rate-limit logs, and can apply field-level transformations (remove, redact, rename, add) based on the matched policy. Policies are watched and reloaded on file change. The transform is gated behind the `transforms-policy` cargo feature. + +authors: jaronoff97 diff --git a/changelog.d/policy_transform_otel_mode.feature.md b/changelog.d/policy_transform_otel_mode.feature.md new file mode 100644 index 0000000000000..de02b6c8674a7 --- /dev/null +++ b/changelog.d/policy_transform_otel_mode.feature.md @@ -0,0 +1,3 @@ +Added `mode: otel` option to the `policy` transform. When enabled, the transform treats each Vector event as an OTLP envelope (`{ resourceLogs: [...] }`) produced by the `opentelemetry` source with `use_otlp_decoding.logs = true`, iterates every `logRecord` inside, and applies policies per-record. Empty `scopeLogs` and `resourceLogs` entries are pruned automatically; if every record is filtered out the entire event is dropped. The default `mode: flat` preserves the previous one-event-per-decision behaviour. + +authors: jaronoff97 diff --git a/deny.toml b/deny.toml index 37db54adf7956..c24527ad03c04 100644 --- a/deny.toml +++ b/deny.toml @@ -51,4 +51,5 @@ ignore = [ { id = "RUSTSEC-2026-0104", reason = "rustls-webpki 0.102/0.101 CRL parsing panic - tonic upgrade (https://github.com/vectordotdev/vector/issues/19179); 0.103 already patched to 0.103.13" }, { id = "RUSTSEC-2026-0105", reason = "core2 is unmaintained and all versions yanked - transitive dependency via libflate -> apache-avro, no safe upgrade available" }, { id = "RUSTSEC-2026-0119", reason = "hickory-proto 0.24 - unpatched crate (https://github.com/mongodb/mongo-rust-driver/pull/1682)" }, + { id = "RUSTSEC-2024-0384", reason = "instant is unmaintained - transitive dependency via policy-rs -> notify 7 -> notify-types, no safe upgrade available upstream" }, ] diff --git a/distribution/docker/tero/Dockerfile b/distribution/docker/tero/Dockerfile new file mode 100644 index 0000000000000..76606ff266dd8 --- /dev/null +++ b/distribution/docker/tero/Dockerfile @@ -0,0 +1,50 @@ +# Tero Vector — runtime image +# +# Receives a prebuilt `vector` binary in the build context and bakes it into +# a minimal Debian-slim runtime. Build orchestration lives in +# `.github/workflows/tero-release.yaml`, which compiles the binary natively +# on amd64 and arm64 runners and supplies the right binary per --platform. +# +# Build args: +# TARGETARCH Set automatically by buildx (`amd64` | `arm64`). Selects +# which prebuilt binary directory to copy from. +# +# Build context layout (relative to context root): +# amd64/vector Linux x86_64 binary +# arm64/vector Linux aarch64 binary +# LICENSE License file (copied for OCI metadata) +# README.md Optional, included for OCI metadata + +FROM docker.io/library/debian:bookworm-slim@sha256:0104b334637a5f19aa9c983a91b54c89887c0984081f2068983107a6f6c21eeb AS runtime + +ARG TARGETARCH + +# Common Vector runtime dependencies plus libstdc++ for Vectorscan. +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + tzdata \ + libsasl2-2 \ + libssl3 \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +LABEL org.opencontainers.image.title="vector" +LABEL org.opencontainers.image.description="Tero distribution of Vector with the policy transform enabled." +LABEL org.opencontainers.image.source="https://github.com/usetero/vector" +LABEL org.opencontainers.image.licenses="MPL-2.0" +LABEL org.opencontainers.image.vendor="Tero" + +# Pull the per-arch binary from the build context. The workflow places +# binaries under `amd64/vector` and `arm64/vector` before invoking buildx. +COPY --chmod=0755 ${TARGETARCH}/vector /usr/bin/vector +COPY LICENSE /usr/share/doc/vector/LICENSE + +# Smoke test — fail the image build if the binary can't even print its version. +RUN ["vector", "--version"] + +# Vector's API listens on 8686 by default. +EXPOSE 8686 + +ENTRYPOINT ["/usr/bin/vector"] diff --git a/distribution/docker/tero/README.md b/distribution/docker/tero/README.md new file mode 100644 index 0000000000000..4b7041ddfe8a6 --- /dev/null +++ b/distribution/docker/tero/README.md @@ -0,0 +1,84 @@ +# Tero Vector distribution + +This directory holds the release machinery for the Tero fork of Vector. +The fork tracks upstream `vectordotdev/vector` and adds the `policy` +transform (see `src/transforms/policy/`); everything else is the +unmodified upstream tree. + +## What's produced + +Each release publishes: + +- A multi-arch Docker image to **`ghcr.io/usetero/vector`** at + `:` and `:latest` covering `linux/amd64` and `linux/arm64`. +- Two binary tarballs attached to the GitHub Release: + - `vector--x86_64-unknown-linux-gnu.tar.gz` + - `vector--aarch64-unknown-linux-gnu.tar.gz` +- A `.sha256` checksum next to each tarball. + +The binaries are built with `--features +"target-,transforms-policy"`, i.e. upstream Vector's standard +feature set for the target triple **plus** our `policy` transform. + +## How a release is cut + +1. Land PRs to `master` using Conventional Commits. `feat:`, `fix:`, + `chore:`, etc. +2. `tero-release-please.yaml` watches `master`. After each merge it opens + (or updates) a single "release PR" that bumps + `.release-please-manifest.json`, `Cargo.toml` version, and + `CHANGELOG.md`. +3. When the release PR is merged, release-please creates the + `v` tag and the corresponding GitHub Release. +4. The tag push triggers `tero-release.yaml`, which builds the binaries + and image and attaches the tarballs to the release. + +## Versioning + +Versions follow `-tero.` (e.g. `0.56.0-tero.1`). The base +`` should match the upstream Vector version this fork is +rebased on. `` is bumped automatically by release-please each release. + +When rebasing onto a newer upstream Vector: + +1. Rebase `master` onto the new upstream tag. +2. Manually edit `.release-please-manifest.json` to reset the base, e.g. + `0.56.0-tero.5` → `0.57.0-tero.0`. +3. Merge a `chore(release): rebase onto upstream vX.Y.Z` commit; the + next release-please PR bumps to `0.57.0-tero.1`. + +## One-time setup: disable upstream release workflows + +Upstream's `release.yml` and `publish.yml` also fire on `v*` tag pushes +and try to publish to channels we don't own (Docker Hub `timberio/vector`, +S3 `packages.timber.io`, Homebrew, the upstream GHCR). They will fail +loudly without those credentials. + +In the fork repository: **Settings → Actions → Workflows → disable**: + +- `Release` (`.github/workflows/release.yml`) +- `Publish` (`.github/workflows/publish.yml`) + +This is settings-only and avoids editing any upstream file. The +disablement persists across rebases. + +If you ever need the upstream pipelines back (e.g. to mirror Vector's +own release artifacts), re-enable them from the same panel. + +## Manual smoke test + +Trigger `tero-release.yaml` from the Actions tab with `workflow_dispatch`: + +- `version`: `0.0.0-tero.dryrun` (or any string) +- `dry_run`: `true` + +This builds the binaries and Docker image without pushing to GHCR or +attaching release assets. + +## Files in this directory + +- `Dockerfile` — Debian-slim runtime that receives a prebuilt `vector` + binary via the build context and bakes it into a multi-arch image. +- `README.md` — this file. + +Build orchestration lives in `.github/workflows/tero-release.yaml`. diff --git a/lib/codecs/src/common/mod.rs b/lib/codecs/src/common/mod.rs index e2d14804f7d3b..adc32341bf5fb 100644 --- a/lib/codecs/src/common/mod.rs +++ b/lib/codecs/src/common/mod.rs @@ -1,3 +1,5 @@ //! A collection of common utility features used by both encoding and decoding logic. pub mod length_delimited; +#[cfg(feature = "opentelemetry")] +pub mod otlp; diff --git a/lib/codecs/src/common/otlp.rs b/lib/codecs/src/common/otlp.rs new file mode 100644 index 0000000000000..fc0ef6f757bf5 --- /dev/null +++ b/lib/codecs/src/common/otlp.rs @@ -0,0 +1,168 @@ +//! Shared helpers for converting OTLP trace/span identifiers between their +//! protobuf wire form (raw `bytes`) and their OTLP/JSON form (hex strings). +//! +//! The protobuf `LogRecord.trace_id` / `Span.span_id` etc. are `bytes`, but the +//! canonical OTLP/JSON representation (and what every other tool in the +//! ecosystem, including downstream policy evaluation, expects) is a lowercase +//! hex string. Vector decodes OTLP protobuf into a JSON-shaped event tree, so we +//! normalize these identifiers to hex on decode and convert them back to raw +//! bytes on encode. Keeping the in-memory representation canonical means a plain +//! JSON serializer round-trips correctly and matchers see the hex string they +//! expect. + +use bytes::Bytes; +use vrl::value::Value; + +/// OTLP identifier fields (proto3 JSON camelCase) that are `bytes` on the wire +/// but hex strings in OTLP/JSON. +const ID_FIELDS: [&str; 3] = ["traceId", "spanId", "parentSpanId"]; + +/// Recursively hex-encode every OTLP id field carried as raw bytes. Called +/// after decoding OTLP protobuf so the event tree holds canonical hex strings. +pub fn hex_encode_ids(value: &mut Value) { + walk(value, true); +} + +/// Recursively hex-decode every OTLP id field carried as a hex string back into +/// raw bytes. Called before encoding to OTLP protobuf. +pub fn hex_decode_ids(value: &mut Value) { + walk(value, false); +} + +fn walk(value: &mut Value, encode: bool) { + match value { + Value::Object(map) => { + for (key, child) in map.iter_mut() { + if ID_FIELDS.contains(&key.as_str()) { + convert(child, encode); + } else { + walk(child, encode); + } + } + } + Value::Array(items) => { + for item in items.iter_mut() { + walk(item, encode); + } + } + _ => {} + } +} + +fn convert(value: &mut Value, encode: bool) { + let Value::Bytes(bytes) = value else { + return; + }; + if encode { + *value = Value::Bytes(Bytes::from(hex_encode(bytes))); + } else if let Some(raw) = hex_decode(bytes) { + *value = Value::Bytes(Bytes::from(raw)); + } +} + +fn hex_encode(bytes: &[u8]) -> Vec { + const DIGITS: &[u8; 16] = b"0123456789abcdef"; + let mut out = Vec::with_capacity(bytes.len() * 2); + for &b in bytes { + out.push(DIGITS[(b >> 4) as usize]); + out.push(DIGITS[(b & 0x0f) as usize]); + } + out +} + +fn hex_decode(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(2) { + return None; + } + let nibble = |c: u8| -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } + }; + let mut out = Vec::with_capacity(bytes.len() / 2); + for pair in bytes.chunks_exact(2) { + out.push((nibble(pair[0])? << 4) | nibble(pair[1])?); + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use vrl::value::Value; + + fn obj(pairs: Vec<(&str, Value)>) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.into(), v)).collect()) + } + + #[test] + fn round_trips_span_id() { + let raw = Bytes::from_static(b"span0001"); + let mut tree = obj(vec![("spanId", Value::Bytes(raw.clone()))]); + hex_encode_ids(&mut tree); + assert_eq!( + tree.as_object().unwrap().get("spanId").unwrap(), + &Value::Bytes(Bytes::from_static(b"7370616e30303031")), + ); + hex_decode_ids(&mut tree); + assert_eq!( + tree.as_object().unwrap().get("spanId").unwrap(), + &Value::Bytes(raw), + ); + } + + #[test] + fn walks_nested_arrays() { + let mut tree = obj(vec![( + "resourceSpans", + Value::Array(vec![obj(vec![( + "spans", + Value::Array(vec![obj(vec![( + "traceId", + Value::Bytes(Bytes::from_static(&[0x12, 0x34])), + )])]), + )])]), + )]); + hex_encode_ids(&mut tree); + let trace_id = tree + .as_object() + .unwrap() + .get("resourceSpans") + .unwrap() + .as_array() + .unwrap()[0] + .as_object() + .unwrap() + .get("spans") + .unwrap() + .as_array() + .unwrap()[0] + .as_object() + .unwrap() + .get("traceId") + .unwrap(); + assert_eq!(trace_id, &Value::Bytes(Bytes::from_static(b"1234"))); + } + + #[test] + fn leaves_attribute_key_named_span_id_untouched() { + // A user attribute whose *key string* is "spanId" lives under the "key" + // field, not as an object field named spanId, so it must not be touched. + let mut tree = obj(vec![( + "attributes", + Value::Array(vec![obj(vec![ + ("key", Value::from("spanId")), + ( + "value", + obj(vec![("stringValue", Value::from("not-an-id"))]), + ), + ])]), + )]); + let before = tree.clone(); + hex_encode_ids(&mut tree); + assert_eq!(tree, before); + } +} diff --git a/lib/codecs/src/decoding/format/otlp.rs b/lib/codecs/src/decoding/format/otlp.rs index 3e81dec089dce..af35d2d2c76e1 100644 --- a/lib/codecs/src/decoding/format/otlp.rs +++ b/lib/codecs/src/decoding/format/otlp.rs @@ -163,20 +163,23 @@ impl Deserializer for OtlpDeserializer { for signal_type in &self.signals { match signal_type { OtlpSignalType::Logs => { - if let Ok(events) = self.logs_deserializer.parse(bytes.clone(), log_namespace) + if let Ok(mut events) = + self.logs_deserializer.parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() && log.get(RESOURCE_LOGS_JSON_FIELD).is_some() { + normalize_ids(&mut events); return Ok(events); } } OtlpSignalType::Metrics => { - if let Ok(events) = self + if let Ok(mut events) = self .metrics_deserializer .parse(bytes.clone(), log_namespace) && let Some(Event::Log(log)) = events.first() && log.get(RESOURCE_METRICS_JSON_FIELD).is_some() { + normalize_ids(&mut events); return Ok(events); } } @@ -188,7 +191,8 @@ impl Deserializer for OtlpDeserializer { && log.get(RESOURCE_SPANS_JSON_FIELD).is_some() { // Convert the log event to a trace event by taking ownership - if let Some(Event::Log(log)) = events.pop() { + if let Some(Event::Log(mut log)) = events.pop() { + crate::common::otlp::hex_encode_ids(log.value_mut()); let trace_event = Event::Trace(log.into()); return Ok(smallvec![trace_event]); } @@ -201,6 +205,17 @@ impl Deserializer for OtlpDeserializer { } } +/// Hex-encode the OTLP id `bytes` fields (`traceId`/`spanId`/`parentSpanId`) of +/// every log event so the in-memory tree carries canonical OTLP/JSON hex +/// strings rather than raw bytes. +fn normalize_ids(events: &mut SmallVec<[Event; 1]>) { + for event in events.iter_mut() { + if let Event::Log(log) = event { + crate::common::otlp::hex_encode_ids(log.value_mut()); + } + } +} + #[cfg(test)] mod tests { use opentelemetry_proto::proto::{ @@ -341,28 +356,28 @@ mod tests { let span = spans.first().expect("should have at least one span"); - // Verify traceId - should be raw bytes (16 bytes for trace_id) + // Verify traceId - normalized to a lowercase hex string (OTLP/JSON form) let trace_id = span .get("traceId") - .and_then(|v| v.as_bytes()) - .expect("traceId should exist and be bytes"); + .and_then(|v| v.as_str()) + .expect("traceId should exist and be a hex string"); assert_eq!( trace_id.as_ref(), - &TEST_TRACE_ID, - "traceId should match the expected 16 bytes (0102030405060708090a0b0c0d0e0f10)" + "0102030405060708090a0b0c0d0e0f10", + "traceId should be the hex encoding of the 16 raw bytes" ); - // Verify spanId - should be raw bytes (8 bytes for span_id) + // Verify spanId - normalized to a lowercase hex string (OTLP/JSON form) let span_id = span .get("spanId") - .and_then(|v| v.as_bytes()) - .expect("spanId should exist and be bytes"); + .and_then(|v| v.as_str()) + .expect("spanId should exist and be a hex string"); assert_eq!( span_id.as_ref(), - &TEST_SPAN_ID, - "spanId should match the expected 8 bytes (0102030405060708)" + "0102030405060708", + "spanId should be the hex encoding of the 8 raw bytes" ); } diff --git a/lib/codecs/src/encoding/format/otlp.rs b/lib/codecs/src/encoding/format/otlp.rs index c3e8d325a8abc..b5c69bd902f81 100644 --- a/lib/codecs/src/encoding/format/otlp.rs +++ b/lib/codecs/src/encoding/format/otlp.rs @@ -97,7 +97,15 @@ impl OtlpSerializer { impl Encoder for OtlpSerializer { type Error = vector_common::Error; - fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + fn encode(&mut self, mut event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + // The in-memory tree carries trace/span ids as hex strings (canonical + // OTLP/JSON). Convert them back to raw `bytes` before protobuf encoding. + match &mut event { + Event::Log(log) => crate::common::otlp::hex_decode_ids(log.value_mut()), + Event::Trace(trace) => crate::common::otlp::hex_decode_ids(trace.value_mut()), + Event::Metric(_) => {} + } + // Determine which descriptor to use based on top-level OTLP fields // This handles events that were decoded with use_otlp_decoding enabled // The deserializer uses use_json_names: true, so fields are in camelCase diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000000000..39c817b1e0e07 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "packages": { + ".": { + "release-type": "rust", + "prerelease": true, + "bump-minor-pre-major": false, + "bump-patch-for-minor-pre-major": false, + "draft": false, + "prerelease-type": "tero", + "include-component-in-tag": false, + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "enhancement", "section": "Enhancements" }, + { "type": "perf", "section": "Performance" }, + { "type": "deps", "section": "Dependencies" }, + { "type": "revert", "section": "Reverts" }, + { "type": "docs", "section": "Documentation", "hidden": true }, + { "type": "chore", "section": "Miscellaneous", "hidden": true }, + { "type": "refactor", "section": "Code Refactoring", "hidden": true }, + { "type": "test", "section": "Tests", "hidden": true }, + { "type": "build", "section": "Build System", "hidden": true }, + { "type": "ci", "section": "Continuous Integration", "hidden": true } + ] + } + } +} diff --git a/src/transforms/mod.rs b/src/transforms/mod.rs index e4f6671828807..e34a98317322a 100644 --- a/src/transforms/mod.rs +++ b/src/transforms/mod.rs @@ -23,6 +23,8 @@ pub mod log_to_metric; pub mod lua; #[cfg(feature = "transforms-metric_to_log")] pub mod metric_to_log; +#[cfg(feature = "transforms-policy")] +pub mod policy; #[cfg(feature = "transforms-remap")] pub mod remap; #[cfg(feature = "transforms-route")] diff --git a/src/transforms/policy/adapter.rs b/src/transforms/policy/adapter.rs new file mode 100644 index 0000000000000..f2f6d8e3711ba --- /dev/null +++ b/src/transforms/policy/adapter.rs @@ -0,0 +1,875 @@ +//! Adapter that exposes a Vector `LogEvent` to the `policy-rs` engine. +//! +//! `policy-rs` evaluates against types that implement its `Matchable` and +//! `Transformable` traits. Vector's `LogEvent` is schema-less, so we wrap a +//! mutable borrow alongside a [`FieldMapping`] and translate every +//! `LogFieldSelector` into an event path on the fly. +//! +//! The adapter is intentionally side-effect free outside the borrow it +//! holds: every read/write goes through `LogEvent`'s own API, so future +//! `policy-rs` versions can change their evaluation strategy without +//! requiring changes anywhere else in Vector. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::LogField; +use policy_rs::{LogFieldSelector, Matchable, Transformable, engine::LogSignal}; +use vector_lib::{ + event::{LogEvent, Value}, + lookup::{OwnedValuePath, PathPrefix}, +}; + +use super::field_mapping::FieldMapping; + +/// Wraps a mutable `LogEvent` and a `FieldMapping` so the `policy-rs` engine +/// can read and mutate it through `Matchable` and `Transformable`. +pub struct VectorLogAdapter<'a> { + log: &'a mut LogEvent, + mapping: &'a FieldMapping, +} + +impl<'a> VectorLogAdapter<'a> { + pub const fn new(log: &'a mut LogEvent, mapping: &'a FieldMapping) -> Self { + Self { log, mapping } + } +} + +/// Resolve a `LogFieldSelector` to the corresponding `LogEvent` value path. +/// +/// Simple fields borrow the mapping's stored path (`Cow::Borrowed`, no +/// allocation); only attribute selectors — which append dynamic segments — +/// build a fresh path (`Cow::Owned`). +/// +/// Returns `None` for selectors the mapping doesn't represent (e.g. the +/// protobuf-default `LogField::Unspecified`); the engine treats `None` as +/// "field not present", the correct fail-soft behaviour for matching. +/// +/// Takes `&FieldMapping` rather than `&self` so callers can hold the resulting +/// borrow of the mapping while mutating `self.log` (disjoint fields). +fn value_path<'m>( + mapping: &'m FieldMapping, + selector: &LogFieldSelector, +) -> Option> { + match selector { + LogFieldSelector::Simple(field) => simple_value_path(mapping, *field).map(Cow::Borrowed), + LogFieldSelector::LogAttribute(path) => Some(Cow::Owned(FieldMapping::append_segments( + &mapping.log_attributes, + path, + ))), + LogFieldSelector::ResourceAttribute(path) => Some(Cow::Owned( + FieldMapping::append_segments(&mapping.resource_attributes, path), + )), + LogFieldSelector::ScopeAttribute(path) => Some(Cow::Owned(FieldMapping::append_segments( + &mapping.scope_attributes, + path, + ))), + } +} + +const fn simple_value_path(mapping: &FieldMapping, field: LogField) -> Option<&OwnedValuePath> { + Some(match field { + LogField::Body => &mapping.body.0, + LogField::SeverityText => &mapping.severity_text.0, + LogField::TraceId => &mapping.trace_id.0, + LogField::SpanId => &mapping.span_id.0, + LogField::EventName => &mapping.event_name.0, + LogField::ResourceSchemaUrl => &mapping.resource_schema_url.0, + LogField::ScopeSchemaUrl => &mapping.scope_schema_url.0, + LogField::Unspecified => return None, + }) +} + +impl Matchable for VectorLogAdapter<'_> { + type Signal = LogSignal; + + fn get_field(&self, field: &LogFieldSelector) -> Option> { + let path = value_path(self.mapping, field)?; + let value = self.log.get((PathPrefix::Event, path.as_ref()))?; + value_to_match_string(value) + } + + fn field_exists(&self, field: &LogFieldSelector) -> bool { + let Some(path) = value_path(self.mapping, field) else { + return false; + }; + self.log.get((PathPrefix::Event, path.as_ref())).is_some() + } +} + +impl Transformable for VectorLogAdapter<'_> { + fn set_field(&mut self, field: &LogFieldSelector, value: &str) { + if let Some(path) = value_path(self.mapping, field) { + self.log + .insert((PathPrefix::Event, path.as_ref()), value.to_string()); + } + } + + fn delete_field(&mut self, field: &LogFieldSelector) -> bool { + let Some(path) = value_path(self.mapping, field) else { + return false; + }; + self.log + .remove((PathPrefix::Event, path.as_ref())) + .is_some() + } + + fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { + // The engine guarantees `from` exists and `to` does not, so we can + // remove from the source and unconditionally insert at the target + // without losing data on a name collision. + let (Some(from_path), Some(to_path)) = + (value_path(self.mapping, from), value_path(self.mapping, to)) + else { + return; + }; + if let Some(value) = self.log.remove((PathPrefix::Event, from_path.as_ref())) { + self.log + .insert((PathPrefix::Event, to_path.as_ref()), value); + } + } +} + +/// Coerce a `LogEvent` value to a string suitable for `policy-rs` pattern +/// matching. +/// +/// Strings (`Bytes` containing valid UTF-8) are borrowed; scalars are +/// stringified. Containers, timestamps, nulls, and regexes can't drive a +/// string match cleanly and return `None` — `field_exists` still reports +/// them as present so `exists: true` matchers fire correctly. +fn value_to_match_string(value: &Value) -> Option> { + match value { + Value::Bytes(bytes) => std::str::from_utf8(bytes).ok().map(Cow::Borrowed), + Value::Integer(i) => Some(Cow::Owned(i.to_string())), + Value::Float(f) => Some(Cow::Owned(f.to_string())), + Value::Boolean(b) => Some(Cow::Borrowed(if *b { "true" } else { "false" })), + Value::Timestamp(_) + | Value::Object(_) + | Value::Array(_) + | Value::Regex(_) + | Value::Null => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use policy_rs::proto::tero::policy::v1::LogField; + use vector_lib::event::LogEvent; + use vector_lib::lookup::OwnedTargetPath; + + fn make_log() -> LogEvent { + let mut log = LogEvent::default(); + log.insert("message", "hello world"); + log.insert("severity_text", "ERROR"); + log.insert("trace_id", "abc123"); + log.insert("span_id", "def456"); + log.insert("event_name", "user.login"); + log.insert( + "resource.schema_url", + "https://opentelemetry.io/schemas/1.0", + ); + log.insert("scope.schema_url", "https://opentelemetry.io/schemas/1.0"); + log.insert("attributes.user_id", "42"); + log.insert("attributes.flagged", true); + log.insert("attributes.ratio", 1.5); + log.insert("resource.attributes.service\\.name", "frontend"); + log.insert("scope.attributes.library", "tracer"); + log + } + + fn mapping() -> FieldMapping { + FieldMapping::default() + } + + // --- Matchable: get_field -------------------------------------------- + + #[test] + fn get_simple_body() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("hello world"), + ); + } + + #[test] + fn get_simple_severity_text() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::SeverityText)) + .as_deref(), + Some("ERROR"), + ); + } + + #[test] + fn get_simple_trace_and_span_ids() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::TraceId)) + .as_deref(), + Some("abc123"), + ); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::SpanId)) + .as_deref(), + Some("def456"), + ); + } + + #[test] + fn get_unspecified_field_returns_none() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::Simple(LogField::Unspecified)), + None, + ); + } + + #[test] + fn get_log_attribute_flat() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("42"), + ); + } + + #[test] + fn get_log_attribute_nested() { + let mut log = LogEvent::default(); + log.insert("attributes.http.method", "GET"); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec![ + "http".to_string(), + "method".to_string() + ])) + .as_deref(), + Some("GET"), + ); + } + + #[test] + fn get_resource_attribute_with_dot_in_key() { + // policy-rs surfaces OTel attributes as single-segment selectors + // even when the OTel key contains dots (e.g. "service.name"). The + // adapter must treat each segment literally — segments containing + // dots are not re-parsed as dotted paths. + // + // Insert through a typed path that mirrors what `append_segments` + // produces, then confirm the adapter can read it back through the + // same selector. + let mut log = LogEvent::default(); + let m = mapping(); + let attr_path = + FieldMapping::append_segments(&m.resource_attributes, &["service.name".to_string()]); + log.insert(&OwnedTargetPath::event(attr_path), "frontend"); + + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])) + .as_deref(), + Some("frontend"), + ); + } + + #[test] + fn get_scope_attribute() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::ScopeAttribute(vec![ + "library".to_string() + ])) + .as_deref(), + Some("tracer"), + ); + } + + #[test] + fn get_integer_value_stringifies() { + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["count".to_string()])) + .as_deref(), + Some("42"), + ); + } + + #[test] + fn get_boolean_value_stringifies() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["flagged".to_string()])) + .as_deref(), + Some("true"), + ); + } + + #[test] + fn get_float_value_stringifies() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + let v = adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["ratio".to_string()])) + .map(Cow::into_owned); + // We don't pin the exact float representation — just confirm it's a string starting with "1.5". + assert!( + matches!(v, Some(ref s) if s.starts_with("1.5")), + "got: {v:?}" + ); + } + + #[test] + fn get_timestamp_returns_none_for_matching() { + let mut log = LogEvent::default(); + log.insert("attributes.ts", Value::Timestamp(Utc::now())); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::LogAttribute(vec!["ts".to_string()])), + None, + ); + } + + #[test] + fn get_missing_field_returns_none() { + let mut log = LogEvent::default(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter.get_field(&LogFieldSelector::Simple(LogField::Body)), + None, + ); + } + + // --- Matchable: field_exists ----------------------------------------- + + #[test] + fn exists_true_for_string_field() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + } + + #[test] + fn exists_true_for_non_string_field() { + // OTel-style: integer/bool/timestamp values should still satisfy + // `exists: true` matchers even though `get_field` returns None. + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + log.insert("attributes.ts", Value::Timestamp(Utc::now())); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".to_string()]))); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["ts".to_string()]))); + } + + #[test] + fn exists_false_for_missing_field() { + let mut log = LogEvent::default(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + assert!( + !adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["missing".to_string()])) + ); + } + + #[test] + fn exists_false_for_unspecified() { + let mut log = make_log(); + let m = mapping(); + let adapter = VectorLogAdapter::new(&mut log, &m); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Unspecified))); + } + + // --- Transformable: set_field ---------------------------------------- + + #[test] + fn set_field_creates_when_absent() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["new_key".to_string()]), + "value", + ); + } + assert_eq!( + log.get("attributes.new_key").and_then(|v| v.as_str()), + Some("value".into()), + ); + } + + #[test] + fn set_field_overwrites_when_present() { + let mut log = make_log(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(LogField::Body), "replaced"); + } + assert_eq!( + log.get("message").and_then(|v| v.as_str()), + Some("replaced".into()), + ); + } + + #[test] + fn set_nested_attribute_creates_intermediate_maps() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::LogAttribute(vec!["http".to_string(), "method".to_string()]), + "POST", + ); + } + assert_eq!( + log.get("attributes.http.method").and_then(|v| v.as_str()), + Some("POST".into()), + ); + } + + #[test] + fn set_unspecified_is_noop() { + let mut log = make_log(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(LogField::Unspecified), "x"); + } + assert_eq!(log, snapshot_before); + } + + // --- Transformable: delete_field ------------------------------------- + + #[test] + fn delete_field_removes_present_field() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + }; + assert!(was_present); + assert!(log.get("attributes.user_id").is_none()); + } + + #[test] + fn delete_field_returns_false_when_absent() { + let mut log = LogEvent::default(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::LogAttribute(vec!["nope".to_string()])) + }; + assert!(!was_present); + } + + #[test] + fn delete_field_returns_false_for_unspecified() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(LogField::Unspecified)) + }; + assert!(!was_present); + } + + #[test] + fn delete_simple_body() { + let mut log = make_log(); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(LogField::Body)) + }; + assert!(was_present); + assert!(log.get("message").is_none()); + } + + // --- Transformable: move_field --------------------------------------- + + #[test] + fn move_field_within_log_attributes() { + let mut log = LogEvent::default(); + log.insert("attributes.usr", "admin"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["usr".to_string()]), + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + ); + } + assert!(log.get("attributes.usr").is_none()); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("admin".into()), + ); + } + + #[test] + fn move_field_preserves_non_string_value() { + let mut log = LogEvent::default(); + log.insert("attributes.count", 42_i64); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["count".to_string()]), + &LogFieldSelector::LogAttribute(vec!["renamed".to_string()]), + ); + } + assert!(log.get("attributes.count").is_none()); + assert_eq!( + log.get("attributes.renamed").and_then(|v| v.as_integer()), + Some(42), + ); + } + + #[test] + fn move_field_absent_source_is_noop() { + let mut log = LogEvent::default(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["missing".to_string()]), + &LogFieldSelector::LogAttribute(vec!["target".to_string()]), + ); + } + assert_eq!(log, snapshot_before); + } + + #[test] + fn move_field_unspecified_endpoints_noop() { + let mut log = make_log(); + let snapshot_before = log.clone(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::Simple(LogField::Unspecified), + &LogFieldSelector::LogAttribute(vec!["x".to_string()]), + ); + adapter.move_field( + &LogFieldSelector::LogAttribute(vec!["user_id".to_string()]), + &LogFieldSelector::Simple(LogField::Unspecified), + ); + } + assert_eq!(log, snapshot_before); + } + + // --- Custom mappings ------------------------------------------------- + + #[test] + fn custom_body_path_is_honored() { + use vector_lib::lookup::lookup_v2::ConfigValuePath; + let mut log = LogEvent::default(); + log.insert("log.body", "custom"); + let mapping = FieldMapping { + body: ConfigValuePath::try_from("log.body".to_string()).unwrap(), + ..FieldMapping::default() + }; + let adapter = VectorLogAdapter::new(&mut log, &mapping); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(LogField::Body)) + .as_deref(), + Some("custom"), + ); + } + + #[test] + fn custom_attribute_root_is_honored() { + use vector_lib::lookup::lookup_v2::ConfigValuePath; + let mut log = LogEvent::default(); + log.insert("data.user_id", "42"); + let mapping = FieldMapping { + log_attributes: ConfigValuePath::try_from("data".to_string()).unwrap(), + ..FieldMapping::default() + }; + let adapter = VectorLogAdapter::new(&mut log, &mapping); + assert_eq!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["user_id".to_string()])) + .as_deref(), + Some("42"), + ); + } + + // --- Simple-field exhaustive coverage -------------------------------- + // + // Every `LogField` variant goes through the same `simple_path` table. + // The cases below confirm get/set/delete behave consistently across + // every variant we expose (Body and SeverityText already had bespoke + // tests; these round out the rest). + + /// One get/delete/set cycle for a simple field. Helper rather than + /// per-variant duplication so future `LogField` additions just append a + /// caller below. + fn assert_simple_round_trip(field: LogField, event_path: &str) { + let m = mapping(); + + // get + let mut log = LogEvent::default(); + log.insert(event_path, "initial"); + { + let adapter = VectorLogAdapter::new(&mut log, &m); + assert_eq!( + adapter + .get_field(&LogFieldSelector::Simple(field)) + .as_deref(), + Some("initial"), + "get failed for {field:?} at {event_path:?}", + ); + } + + // delete + let was_present = { + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::Simple(field)) + }; + assert!(was_present, "delete returned false for {field:?}"); + assert!( + log.get(event_path).is_none(), + "delete left {event_path:?} present for {field:?}", + ); + + // set + { + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field(&LogFieldSelector::Simple(field), "after-set"); + } + assert_eq!( + log.get(event_path).and_then(|v| v.as_str()), + Some("after-set".into()), + "set didn't insert at {event_path:?} for {field:?}", + ); + } + + #[test] + fn round_trip_simple_body() { + assert_simple_round_trip(LogField::Body, "message"); + } + + #[test] + fn round_trip_simple_severity_text() { + assert_simple_round_trip(LogField::SeverityText, "severity_text"); + } + + #[test] + fn round_trip_simple_trace_id() { + assert_simple_round_trip(LogField::TraceId, "trace_id"); + } + + #[test] + fn round_trip_simple_span_id() { + assert_simple_round_trip(LogField::SpanId, "span_id"); + } + + #[test] + fn round_trip_simple_event_name() { + assert_simple_round_trip(LogField::EventName, "event_name"); + } + + #[test] + fn round_trip_simple_resource_schema_url() { + assert_simple_round_trip(LogField::ResourceSchemaUrl, "resource.schema_url"); + } + + #[test] + fn round_trip_simple_scope_schema_url() { + assert_simple_round_trip(LogField::ScopeSchemaUrl, "scope.schema_url"); + } + + // --- Resource / scope attribute Transformable coverage --------------- + // + // `LogAttribute` is already covered in detail. These mirror the same + // operations for the resource and scope namespaces so the namespace + // dispatch in `path_for` is exercised end-to-end. + + #[test] + fn set_resource_attribute_flat() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["region".to_string()]), + "us-east-1", + ); + } + assert_eq!( + log.get("resource.attributes.region") + .and_then(|v| v.as_str()), + Some("us-east-1".into()), + ); + } + + #[test] + fn set_resource_attribute_nested() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["k8s".to_string(), "pod".to_string()]), + "vector-0", + ); + } + assert_eq!( + log.get("resource.attributes.k8s.pod") + .and_then(|v| v.as_str()), + Some("vector-0".into()), + ); + } + + #[test] + fn delete_resource_attribute() { + let mut log = LogEvent::default(); + log.insert("resource.attributes.region", "us-east-1"); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::ResourceAttribute(vec![ + "region".to_string(), + ])) + }; + assert!(was_present); + assert!(log.get("resource.attributes.region").is_none()); + } + + #[test] + fn move_resource_attribute_within_namespace() { + let mut log = LogEvent::default(); + log.insert("resource.attributes.zone", "ap-south-1a"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::ResourceAttribute(vec!["zone".to_string()]), + &LogFieldSelector::ResourceAttribute(vec!["availability_zone".to_string()]), + ); + } + assert!(log.get("resource.attributes.zone").is_none()); + assert_eq!( + log.get("resource.attributes.availability_zone") + .and_then(|v| v.as_str()), + Some("ap-south-1a".into()), + ); + } + + #[test] + fn set_scope_attribute_flat() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ScopeAttribute(vec!["lib_version".to_string()]), + "1.2.3", + ); + } + assert_eq!( + log.get("scope.attributes.lib_version") + .and_then(|v| v.as_str()), + Some("1.2.3".into()), + ); + } + + #[test] + fn set_scope_attribute_nested() { + let mut log = LogEvent::default(); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.set_field( + &LogFieldSelector::ScopeAttribute(vec!["vendor".to_string(), "name".to_string()]), + "otel", + ); + } + assert_eq!( + log.get("scope.attributes.vendor.name") + .and_then(|v| v.as_str()), + Some("otel".into()), + ); + } + + #[test] + fn delete_scope_attribute() { + let mut log = LogEvent::default(); + log.insert("scope.attributes.lib_version", "1.2.3"); + let was_present = { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.delete_field(&LogFieldSelector::ScopeAttribute(vec![ + "lib_version".to_string(), + ])) + }; + assert!(was_present); + assert!(log.get("scope.attributes.lib_version").is_none()); + } + + #[test] + fn move_scope_attribute_within_namespace() { + let mut log = LogEvent::default(); + log.insert("scope.attributes.legacy", "x"); + { + let m = mapping(); + let mut adapter = VectorLogAdapter::new(&mut log, &m); + adapter.move_field( + &LogFieldSelector::ScopeAttribute(vec!["legacy".to_string()]), + &LogFieldSelector::ScopeAttribute(vec!["renamed".to_string()]), + ); + } + assert!(log.get("scope.attributes.legacy").is_none()); + assert_eq!( + log.get("scope.attributes.renamed").and_then(|v| v.as_str()), + Some("x".into()), + ); + } +} diff --git a/src/transforms/policy/config.rs b/src/transforms/policy/config.rs new file mode 100644 index 0000000000000..4e87ba0116ddc --- /dev/null +++ b/src/transforms/policy/config.rs @@ -0,0 +1,355 @@ +//! Configuration for the `policy` transform. + +use std::cell::RefCell; +use std::sync::Arc; + +use policy_rs::{ + PolicyEngine, PolicyRegistry, + config::{FileProviderConfig, ProviderConfig as PolicyRsProviderConfig, register_providers}, +}; +use serde::{Deserialize, Serialize}; +use vector_lib::{ + config::clone_input_definitions, + configurable::{ + Configurable, GenerateError, Metadata, ToValue, configurable_component, + schema::{SchemaGenerator, SchemaObject}, + }, +}; + +use crate::{ + config::{ + DataType, GenerateConfig, Input, OutputId, TransformConfig, TransformContext, + TransformOutput, + }, + schema, + transforms::Transform, +}; + +use super::field_mapping::FieldMapping; +use super::transform::Policy; + +/// Configuration for the `policy` transform. +/// +/// The transform delegates filtering, sampling, rate-limiting, and field +/// transformation to the external [`policy-rs`](https://github.com/usetero/policy-rs) +/// engine. Policies are loaded from configured policy providers and +/// reloaded automatically when the providers report changes. +#[configurable_component(transform( + "policy", + "Evaluate log events against a policy file and apply the resulting keep/drop/sample/rate-limit and field-transform actions." +))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct PolicyConfig { + /// Policy providers to register with the policy registry. + /// + /// Each provider uses policy-rs' tagged provider configuration format, + /// such as `{ type = "file", id = "local", path = "/etc/vector/policies.json" }`. + #[configurable(derived)] + pub policy_providers: Vec, + + /// How Vector events are mapped to policy-rs records. + /// + /// `flat` (the default) treats every Vector event as a single log + /// record and uses the configurable `field_mapping`. + /// + /// `otel` treats every Vector event as an OTLP envelope and iterates + /// `resourceLogs[].scopeLogs[].logRecords[]` internally, applying + /// policies per-record and pruning empty children. `field_mapping` + /// is ignored in this mode — OTLP field locations are fixed by the + /// protocol. + #[configurable(derived)] + #[serde(default)] + pub mode: PolicyMode, + + /// Mapping between `policy-rs` log field selectors and paths within a + /// Vector `LogEvent`. Only used when `mode = flat`. + /// + /// Defaults follow OpenTelemetry semantic conventions so logs produced + /// by the `opentelemetry` source are matched without further + /// configuration. + #[configurable(derived)] + #[serde(default)] + pub field_mapping: FieldMapping, +} + +/// Iteration mode for the `policy` transform. +/// +/// NOTE: the two modes intentionally use different matching semantics, because +/// they target different data models — `flat` wraps Vector's schema-less +/// `LogEvent`, while `otel` follows the OTLP/JSON spec exactly (matching the +/// `policy-rs` conformance suite). Two differences are worth knowing when +/// moving a policy between modes: +/// +/// * Non-string values: in `flat` mode an integer/float/boolean attribute is +/// stringified and is therefore matchable (and redactable); in `otel` mode +/// only a `stringValue` is matchable — other `AnyValue` variants satisfy +/// `exists` but never a regex/exact match. +/// * Empty values: `otel` mode treats an empty string / valueless `AnyValue` +/// as absent (a `""` body does not `exist`); `flat` mode treats any present +/// value as present. +#[configurable_component] +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum PolicyMode { + /// One Vector event maps to one log record. Use this mode for sources + /// that emit flat Vector events (the default for most sources). + /// + /// Policy field selectors are resolved through `field_mapping`. + #[default] + Flat, + + /// One Vector event is an OTLP envelope (`{ resourceLogs: [...] }`). + /// + /// The transform iterates every `logRecord` inside the envelope, + /// applies policies per-record, and prunes empty `scopeLogs` and + /// `resourceLogs` entries. If every record is filtered out, the + /// entire envelope event is dropped. + /// + /// Use this mode with Vector's `opentelemetry` source when + /// `use_otlp_decoding.logs` is `true`. + Otel, +} + +/// Policy provider configuration. +/// +/// This wraps policy-rs' provider configuration so Vector can embed the +/// library's tagged provider enum directly in the transform config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(transparent)] +pub struct PolicyProviderConfig(PolicyRsProviderConfig); + +impl PolicyProviderConfig { + pub(super) fn file(id: impl Into, path: impl Into) -> Self { + Self(PolicyRsProviderConfig::File(FileProviderConfig { + id: id.into(), + path: path.into(), + })) + } + + fn into_inner(self) -> PolicyRsProviderConfig { + self.0 + } +} + +impl Configurable for PolicyProviderConfig { + fn metadata() -> Metadata { + let mut metadata = Metadata::default(); + metadata.set_description( + "Policy provider configuration using policy-rs' tagged provider format.", + ); + metadata + } + + fn generate_schema(_: &RefCell) -> Result { + Ok(SchemaObject::default()) + } +} + +impl ToValue for PolicyProviderConfig { + fn to_value(&self) -> serde_json::Value { + serde_json::to_value(self).expect("Could not convert policy provider config to JSON") + } +} + +impl GenerateConfig for PolicyConfig { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self { + policy_providers: vec![PolicyProviderConfig::file( + "local", + "/etc/vector/policies.json", + )], + mode: PolicyMode::default(), + field_mapping: FieldMapping::default(), + }) + .unwrap() + } +} + +impl PolicyConfig { + fn provider_configs(&self) -> crate::Result> { + if !self.policy_providers.is_empty() { + return Ok(self + .policy_providers + .iter() + .cloned() + .map(PolicyProviderConfig::into_inner) + .collect()); + } + + Err("policy transform requires at least one policy provider".into()) + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "policy")] +impl TransformConfig for PolicyConfig { + async fn build(&self, _context: &TransformContext) -> crate::Result { + let registry = PolicyRegistry::new(); + let provider_configs = self.provider_configs()?; + register_providers(&provider_configs, ®istry) + .await + .map_err(|error| format!("failed to register policy providers: {error}"))?; + + let policy = Policy::new( + Arc::new(registry), + Arc::new(PolicyEngine::new()), + Arc::new(self.field_mapping.clone()), + self.mode, + ); + Ok(Transform::event_task(policy)) + } + + fn input(&self) -> Input { + // Accept everything; metrics and traces pass through unchanged. + Input::all() + } + + fn outputs( + &self, + _: &TransformContext, + input_definitions: &[(OutputId, schema::Definition)], + ) -> Vec { + vec![TransformOutput::new( + DataType::all_bits(), + clone_input_definitions(input_definitions), + )] + } + + fn enable_concurrency(&self) -> bool { + // Rate-limit state lives inside a single `PolicyEngine` instance, so + // running multiple copies of the transform would split the + // per-window counters and silently raise the effective rate limit. + // Keep concurrency disabled until policy-rs offers a shared + // rate-limiter handle. + false + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn generate_config() { + crate::test_util::test_generate_config::(); + } + + #[test] + fn deserialize_minimal() { + let config: PolicyConfig = toml::from_str( + r#" +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" +"#, + ) + .unwrap(); + let providers = config.provider_configs().unwrap(); + assert_eq!(providers.len(), 1); + assert!(matches!(providers[0], PolicyRsProviderConfig::File(_))); + assert_eq!(config.field_mapping, FieldMapping::default()); + } + + #[test] + fn deserialize_with_field_mapping_overrides() { + let config: PolicyConfig = toml::from_str( + r#" +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" + +[field_mapping] +body = "log.body" +"#, + ) + .unwrap(); + assert_eq!(String::from(config.field_mapping.body.clone()), "log.body"); + } + + #[test] + fn deserialize_rejects_unknown_field() { + // The unknown top-level key must come BEFORE the `[[policy_providers]]` + // header — otherwise TOML attaches it to the array element instead of + // the root table, which would slip past `deny_unknown_fields` on + // `PolicyConfig`. + let result: Result = toml::from_str( + r#" +unknown = "value" + +[[policy_providers]] +id = "local" +type = "file" +path = "/tmp/policies.json" +"#, + ); + assert!( + result.is_err(), + "unknown top-level fields should be rejected" + ); + } + + #[test] + fn deserialize_requires_policy_providers() { + let result: Result = toml::from_str(""); + assert!(result.is_err(), "policy_providers is required"); + } + + #[test] + fn empty_config_has_no_provider_configs() { + let config: PolicyConfig = toml::from_str( + r#" +policy_providers = [] +"#, + ) + .unwrap(); + let result = config.provider_configs(); + assert!(result.is_err(), "policy_providers must not be empty"); + } + + #[test] + fn deserialize_multiple_policy_providers() { + let config: PolicyConfig = toml::from_str( + r#" +[[policy_providers]] +id = "primary" +type = "file" +path = "/etc/vector/primary.json" + +[[policy_providers]] +id = "secondary" +type = "file" +path = "/etc/vector/secondary.json" +"#, + ) + .unwrap(); + let providers = config.provider_configs().unwrap(); + assert_eq!(providers.len(), 2); + assert!( + providers + .iter() + .all(|p| matches!(p, PolicyRsProviderConfig::File(_))) + ); + } + + #[test] + fn deserialize_rejects_unknown_provider_type() { + // policy-rs' `ProviderConfig` is a tagged enum on the `type` key. + // An unknown discriminator must fail to deserialize so users get a + // clear error rather than silently dropping the provider. + let result: Result = toml::from_str( + r#" +[[policy_providers]] +id = "bogus" +type = "nonexistent-provider-type" +path = "/dev/null" +"#, + ); + assert!( + result.is_err(), + "unknown provider `type` value should be rejected", + ); + } +} diff --git a/src/transforms/policy/field_mapping.rs b/src/transforms/policy/field_mapping.rs new file mode 100644 index 0000000000000..c504c437d7015 --- /dev/null +++ b/src/transforms/policy/field_mapping.rs @@ -0,0 +1,182 @@ +//! Configurable mapping between `policy-rs` log field selectors and Vector +//! `LogEvent` paths. +//! +//! Vector's `LogEvent` is schema-less: the `policy` transform must therefore +//! be told where in each event the body, severity, attributes, etc. live. +//! Defaults follow OpenTelemetry semantic conventions so that events emitted +//! by Vector's `opentelemetry` source work out of the box. + +use vector_lib::{ + configurable::configurable_component, + lookup::lookup_v2::{ConfigValuePath, OwnedValuePath}, +}; + +/// Maps `policy-rs` log field selectors onto paths inside a Vector `LogEvent`. +/// +/// All defaults follow OpenTelemetry semantic conventions so that logs emitted +/// by the `opentelemetry` source are matched without additional configuration. +#[configurable_component] +#[derive(Clone, Debug, PartialEq, Eq)] +#[serde(deny_unknown_fields, default)] +pub struct FieldMapping { + /// Path to the log body (`LogField::Body`). Default: `.message`. + pub body: ConfigValuePath, + + /// Path to the severity-text field (`LogField::SeverityText`). Default: `.severity_text`. + pub severity_text: ConfigValuePath, + + /// Path to the trace ID field (`LogField::TraceId`). Default: `.trace_id`. + pub trace_id: ConfigValuePath, + + /// Path to the span ID field (`LogField::SpanId`). Default: `.span_id`. + pub span_id: ConfigValuePath, + + /// Path to the event name field (`LogField::EventName`). Default: `.event_name`. + pub event_name: ConfigValuePath, + + /// Path to the resource schema URL (`LogField::ResourceSchemaUrl`). Default: `.resource.schema_url`. + pub resource_schema_url: ConfigValuePath, + + /// Path to the scope schema URL (`LogField::ScopeSchemaUrl`). Default: `.scope.schema_url`. + pub scope_schema_url: ConfigValuePath, + + /// Root path for log attributes (`LogFieldSelector::LogAttribute`). Default: `.attributes`. + /// + /// The selector's path segments are appended below this root to form the full event path. + pub log_attributes: ConfigValuePath, + + /// Root path for resource attributes (`LogFieldSelector::ResourceAttribute`). Default: `.resource.attributes`. + pub resource_attributes: ConfigValuePath, + + /// Root path for scope attributes (`LogFieldSelector::ScopeAttribute`). Default: `.scope.attributes`. + pub scope_attributes: ConfigValuePath, +} + +impl Default for FieldMapping { + fn default() -> Self { + Self { + body: parse("message"), + severity_text: parse("severity_text"), + trace_id: parse("trace_id"), + span_id: parse("span_id"), + event_name: parse("event_name"), + resource_schema_url: parse("resource.schema_url"), + scope_schema_url: parse("scope.schema_url"), + log_attributes: parse("attributes"), + resource_attributes: parse("resource.attributes"), + scope_attributes: parse("scope.attributes"), + } + } +} + +impl FieldMapping { + /// Compose `root` with `segments`, returning a fresh `OwnedValuePath`. + /// + /// Used to map e.g. `LogAttribute(["http", "method"])` to + /// `.attributes.http.method` under the default mapping. + pub(crate) fn append_segments(root: &ConfigValuePath, segments: &[String]) -> OwnedValuePath { + let mut path = root.0.clone(); + for segment in segments { + path = path.with_field_appended(segment); + } + path + } +} + +fn parse(path: &str) -> ConfigValuePath { + // SAFETY: the inputs here are static, fully-controlled strings. A panic + // would be a developer error caught by the unit tests below, never a + // runtime input. + ConfigValuePath::try_from(path.to_string()) + .unwrap_or_else(|err| panic!("invalid default path {path:?}: {err}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_paths_render_as_expected() { + let m = FieldMapping::default(); + assert_eq!(String::from(m.body.clone()), "message"); + assert_eq!(String::from(m.severity_text.clone()), "severity_text"); + assert_eq!(String::from(m.trace_id.clone()), "trace_id"); + assert_eq!(String::from(m.span_id.clone()), "span_id"); + assert_eq!(String::from(m.event_name.clone()), "event_name"); + assert_eq!( + String::from(m.resource_schema_url.clone()), + "resource.schema_url" + ); + assert_eq!(String::from(m.scope_schema_url.clone()), "scope.schema_url"); + assert_eq!(String::from(m.log_attributes.clone()), "attributes"); + assert_eq!( + String::from(m.resource_attributes.clone()), + "resource.attributes" + ); + assert_eq!(String::from(m.scope_attributes.clone()), "scope.attributes"); + } + + #[test] + fn append_single_segment() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.log_attributes, &["user_id".to_string()]); + assert_eq!(path.to_string(), "attributes.user_id"); + } + + #[test] + fn append_nested_segments() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments( + &m.log_attributes, + &["http".to_string(), "method".to_string()], + ); + assert_eq!(path.to_string(), "attributes.http.method"); + } + + #[test] + fn append_no_segments_returns_root() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.resource_attributes, &[]); + assert_eq!(path.to_string(), "resource.attributes"); + } + + #[test] + fn segment_with_special_chars_is_quoted() { + let m = FieldMapping::default(); + let path = FieldMapping::append_segments(&m.log_attributes, &["http.method".to_string()]); + // The dot inside the segment must be escaped/quoted in the rendered path. + assert_eq!(path.to_string(), "attributes.\"http.method\""); + } + + #[test] + fn default_round_trips_through_serde() { + let original = FieldMapping::default(); + let json = serde_json::to_string(&original).expect("serialize"); + let parsed: FieldMapping = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(parsed, original); + } + + #[test] + fn empty_object_uses_defaults() { + let parsed: FieldMapping = serde_json::from_str("{}").expect("deserialize {}"); + assert_eq!(parsed, FieldMapping::default()); + } + + #[test] + fn partial_override_keeps_other_defaults() { + let parsed: FieldMapping = + serde_json::from_str(r#"{"body":"log.body"}"#).expect("deserialize"); + assert_eq!(String::from(parsed.body), "log.body"); + // Other fields fall back to defaults. + assert_eq!( + String::from(parsed.severity_text), + String::from(FieldMapping::default().severity_text) + ); + } + + #[test] + fn unknown_field_is_rejected() { + let result: Result = serde_json::from_str(r#"{"unknown":"x"}"#); + assert!(result.is_err(), "unknown fields should be rejected"); + } +} diff --git a/src/transforms/policy/internal_events.rs b/src/transforms/policy/internal_events.rs new file mode 100644 index 0000000000000..8bf3516fbe147 --- /dev/null +++ b/src/transforms/policy/internal_events.rs @@ -0,0 +1,121 @@ +//! Internal events emitted by the `policy` transform. +//! +//! These live inside the transform module so the file under +//! `src/internal_events/` stays untouched — any future schema change can be +//! confined to this one file. + +use vector_lib::internal_event::{ComponentEventsDropped, INTENTIONAL}; + +/// Reason tag attached to `component_discarded_events_total` for events the +/// policy engine declined to forward. +#[derive(Debug, Clone, Copy)] +pub(crate) enum DropReason { + /// A matching policy's `keep` action is `"none"` — drop unconditionally. + PolicyDrop, + /// A percentage-sample policy decided not to keep this event. + SampleRejected, + /// A rate-limited policy's window quota is exhausted. + RateLimited, +} + +impl DropReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + DropReason::PolicyDrop => "policy_drop", + DropReason::SampleRejected => "policy_sample_rejected", + DropReason::RateLimited => "policy_rate_limited", + } + } +} + +/// Per-envelope accumulator for dropped-record counts. +/// +/// OTLP envelopes can drop many records in a single event; emitting one +/// `ComponentEventsDropped` per record would hammer the metrics path. Instead +/// callers `record` each drop and `emit` once per envelope, collapsing N +/// `emit!` calls into at most one per reason. +#[derive(Default)] +pub(crate) struct DropCounts { + policy_drop: u64, + sample_rejected: u64, + rate_limited: u64, +} + +impl DropCounts { + pub(crate) const fn record(&mut self, reason: DropReason) { + match reason { + DropReason::PolicyDrop => self.policy_drop += 1, + DropReason::SampleRejected => self.sample_rejected += 1, + DropReason::RateLimited => self.rate_limited += 1, + } + } + + pub(crate) fn emit(&self) { + emit_dropped(DropReason::PolicyDrop, self.policy_drop); + emit_dropped(DropReason::SampleRejected, self.sample_rejected); + emit_dropped(DropReason::RateLimited, self.rate_limited); + } +} + +/// Emit a single aggregated `ComponentEventsDropped` for `count` records. A +/// zero count emits nothing. +pub(crate) fn emit_dropped(reason: DropReason, count: u64) { + if count == 0 { + return; + } + emit!(ComponentEventsDropped:: { + count: count as usize, + reason: reason.as_str(), + }); +} + +/// Per-envelope accumulator for fail-open evaluation errors. +/// +/// The transform fails open — an evaluation error passes the record through +/// untouched rather than dropping it. Logging one `error!` per record would +/// spam under a systematic failure (e.g. a single malformed input replayed +/// across an envelope), so we record the first error and a count and emit one +/// line per envelope. +#[derive(Default)] +pub(crate) struct EvalErrors { + count: u64, + first: Option, +} + +impl EvalErrors { + pub(crate) fn record(&mut self, error: &dyn std::fmt::Display) { + if self.first.is_none() { + self.first = Some(error.to_string()); + } + self.count += 1; + } + + pub(crate) fn emit(&self) { + if self.count == 0 { + return; + } + error!( + message = + "Policy evaluation failed; affected OTLP records were passed through unchanged.", + count = self.count, + error = self.first.as_deref().unwrap_or_default(), + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reason_strings_are_stable() { + // These tag values flow into emitted metrics. Changing them would be + // a user-visible break, so the test exists to draw attention to it. + assert_eq!(DropReason::PolicyDrop.as_str(), "policy_drop"); + assert_eq!( + DropReason::SampleRejected.as_str(), + "policy_sample_rejected" + ); + assert_eq!(DropReason::RateLimited.as_str(), "policy_rate_limited"); + } +} diff --git a/src/transforms/policy/mod.rs b/src/transforms/policy/mod.rs new file mode 100644 index 0000000000000..5c0326235beb5 --- /dev/null +++ b/src/transforms/policy/mod.rs @@ -0,0 +1,21 @@ +//! Transform that delegates filtering, sampling, rate-limiting, and field +//! transformation to the [`policy-rs`](https://github.com/usetero/policy-rs) +//! library. +//! +//! The transform is fully self-contained inside this directory — no other +//! file in the Vector tree depends on `policy-rs` types — so future updates +//! to the library are isolated to this module. + +mod adapter; +mod field_mapping; +mod internal_events; +mod otlp_common; +mod otlp_log_adapter; +mod otlp_metric_adapter; +mod otlp_trace_adapter; + +pub mod config; +pub mod transform; + +#[cfg(test)] +mod tests; diff --git a/src/transforms/policy/otlp_common.rs b/src/transforms/policy/otlp_common.rs new file mode 100644 index 0000000000000..a0c5a28998257 --- /dev/null +++ b/src/transforms/policy/otlp_common.rs @@ -0,0 +1,278 @@ +//! Shared OTLP value helpers used by the log, metric, and trace adapters. +//! +//! These operate on the JSON-shaped `Value` tree the `opentelemetry` source +//! produces (camelCase keys, `AnyValue`-wrapped attribute values, attributes as +//! `{ key, value }` arrays). They are the read-side primitives — attribute +//! lookup, `AnyValue` coercion, and the envelope lift/prune helpers — that all +//! three signal adapters need; per-signal matching/transform logic lives in the +//! respective `otlp_*_adapter` module. + +use std::borrow::Cow; + +use vector_lib::event::Value; + +// ============================================================================= +// AnyValue coercion. +// ============================================================================= + +/// Coerce a plain string `Value` to a non-empty string. Empty strings count +/// as absent, matching the conformance reference adapter's `non_empty` helper. +pub(super) fn non_empty(value: Option<&Value>) -> Option> { + match value?.as_str() { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +/// Coerce an OTLP `AnyValue` to a string for matching. Only a non-empty +/// `stringValue` is matchable; all other variants (`intValue`, `boolValue`, +/// `arrayValue`, …) return `None` so string matchers and regex redaction +/// never operate on them. +pub(super) fn any_value_string(value: Option<&Value>) -> Option> { + let obj = value?.as_object()?; + match obj.get("stringValue").and_then(Value::as_str) { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +/// Whether an `AnyValue` carries any value variant at all. Powers +/// `exists: true` matchers for attributes whose value isn't a string. +fn any_value_present(value: Option<&Value>) -> bool { + const VARIANTS: [&str; 7] = [ + "stringValue", + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + ]; + match value.and_then(Value::as_object) { + Some(obj) => VARIANTS.iter().any(|k| obj.get(*k).is_some()), + None => false, + } +} + +// ============================================================================= +// Attribute lookup (walks nested kvlistValue). +// ============================================================================= + +/// Resolve an attribute path against an attributes array, walking nested +/// `kvlistValue` entries for multi-segment paths. Returns the leaf's matchable +/// string value, if any. +pub(super) fn find_attribute_path<'a>( + attrs: Option<&'a Value>, + path: &[String], +) -> Option> { + let array = attrs?.as_array()?; + find_in_kvlist(array, path) +} + +fn find_in_kvlist<'a>(attrs: &'a [Value], path: &[String]) -> Option> { + let first = path.first()?; + for kv in attrs { + if !attribute_key_eq(kv, first) { + continue; + } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_string(value); + } + return nested_values(value).and_then(|nested| find_in_kvlist(nested, &path[1..])); + } + None +} + +/// Whether an attribute path resolves to a present value (any `AnyValue` +/// variant), walking nested `kvlistValue` entries. +pub(super) fn attribute_exists_path(attrs: Option<&Value>, path: &[String]) -> bool { + let Some(array) = attrs.and_then(Value::as_array) else { + return false; + }; + exists_in_kvlist(array, path) +} + +fn exists_in_kvlist(attrs: &[Value], path: &[String]) -> bool { + let Some(first) = path.first() else { + return false; + }; + for kv in attrs { + if !attribute_key_eq(kv, first) { + continue; + } + let value = kv.as_object().and_then(|o| o.get("value")); + if path.len() == 1 { + return any_value_present(value); + } + return nested_values(value) + .map(|nested| exists_in_kvlist(nested, &path[1..])) + .unwrap_or(false); + } + false +} + +/// Unwrap an `AnyValue`'s `kvlistValue.values` array. +fn nested_values(value: Option<&Value>) -> Option<&[Value]> { + value + .and_then(Value::as_object) + .and_then(|o| o.get("kvlistValue")) + .and_then(Value::as_object) + .and_then(|o| o.get("values")) + .and_then(Value::as_array) +} + +/// Whether an attribute entry's `key` equals `key`, comparing raw bytes so we +/// skip UTF-8 validation on every entry of a linear scan (the key is an OTLP +/// `string`, but for an equality test the byte representation is sufficient). +pub(super) fn attribute_key_eq(item: &Value, key: &str) -> bool { + matches!( + item.as_object().and_then(|o| o.get("key")), + Some(Value::Bytes(b)) if b.as_ref() == key.as_bytes() + ) +} + +// ============================================================================= +// Envelope iteration helpers. +// ============================================================================= + +/// Remove an object child by key and return it (a move, not a clone). Used to +/// lift `resource` / `scope` out of an envelope entry so they can be borrowed +/// alongside a mutable borrow of a sibling array. +pub(super) fn lift_child(entry: &mut Value, key: &str) -> Option { + entry.as_object_mut().and_then(|o| o.remove(key)) +} + +/// Re-attach a previously [`lift_child`]ed value under `key`. +pub(super) fn reattach_child(entry: &mut Value, key: &str, child: Value) { + if let Some(obj) = entry.as_object_mut() { + obj.insert(key.into(), child); + } +} + +/// Whether `entry`'s array field at `key` is empty or absent — i.e. the entry +/// should be pruned from its parent. +pub(super) fn array_field_is_empty(entry: &Value, key: &str) -> bool { + entry + .as_object() + .and_then(|o| o.get(key)) + .and_then(Value::as_array) + .map(|a| a.is_empty()) + .unwrap_or(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn v(value: serde_json::Value) -> Value { + Value::from(value) + } + + #[test] + fn non_empty_treats_blank_as_absent() { + assert_eq!(non_empty(Some(&v(json!("x")))).as_deref(), Some("x")); + assert_eq!(non_empty(Some(&v(json!("")))), None); + assert_eq!(non_empty(None), None); + } + + #[test] + fn any_value_string_only_matches_non_empty_string() { + assert_eq!( + any_value_string(Some(&v(json!({"stringValue": "hi"})))).as_deref(), + Some("hi"), + ); + assert_eq!(any_value_string(Some(&v(json!({"stringValue": ""})))), None); + assert_eq!(any_value_string(Some(&v(json!({"intValue": "42"})))), None); + assert_eq!(any_value_string(Some(&v(json!({"boolValue": true})))), None); + assert_eq!(any_value_string(None), None); + } + + #[test] + fn find_attribute_path_flat_and_nested() { + let attrs = v(json!([ + {"key": "user_id", "value": {"stringValue": "42"}}, + {"key": "http", "value": {"kvlistValue": {"values": [ + {"key": "method", "value": {"stringValue": "GET"}} + ]}}} + ])); + assert_eq!( + find_attribute_path(Some(&attrs), &["user_id".to_string()]).as_deref(), + Some("42"), + ); + assert_eq!( + find_attribute_path(Some(&attrs), &["http".to_string(), "method".to_string()]) + .as_deref(), + Some("GET"), + ); + // Missing top-level key, and a missing nested segment, both resolve to None. + assert_eq!( + find_attribute_path(Some(&attrs), &["nope".to_string()]), + None + ); + assert_eq!( + find_attribute_path(Some(&attrs), &["http".to_string(), "nope".to_string()]), + None, + ); + } + + #[test] + fn find_attribute_path_non_string_leaf_is_none() { + let attrs = v(json!([{"key": "count", "value": {"intValue": "42"}}])); + assert_eq!( + find_attribute_path(Some(&attrs), &["count".to_string()]), + None + ); + } + + #[test] + fn attribute_exists_path_covers_non_string_and_nested() { + let attrs = v(json!([ + {"key": "count", "value": {"intValue": "42"}}, + {"key": "http", "value": {"kvlistValue": {"values": [ + {"key": "method", "value": {"stringValue": "GET"}} + ]}}} + ])); + // A non-string value still satisfies `exists`. + assert!(attribute_exists_path(Some(&attrs), &["count".to_string()])); + assert!(attribute_exists_path( + Some(&attrs), + &["http".to_string(), "method".to_string()] + )); + assert!(!attribute_exists_path( + Some(&attrs), + &["missing".to_string()] + )); + assert!(!attribute_exists_path(None, &["count".to_string()])); + } + + #[test] + fn attribute_key_eq_compares_key_bytes() { + let kv = v(json!({"key": "service.name", "value": {"stringValue": "api"}})); + assert!(attribute_key_eq(&kv, "service.name")); + assert!(!attribute_key_eq(&kv, "service")); + // Non-object entries never match. + assert!(!attribute_key_eq(&v(json!("x")), "x")); + } + + #[test] + fn lift_and_reattach_round_trip() { + let mut entry = v(json!({"resource": {"attributes": []}, "scopeLogs": []})); + let lifted = lift_child(&mut entry, "resource").expect("resource lifted"); + assert!(entry.as_object().unwrap().get("resource").is_none()); + reattach_child(&mut entry, "resource", lifted); + assert!(entry.as_object().unwrap().get("resource").is_some()); + // Lifting an absent child is a no-op returning None. + assert!(lift_child(&mut entry, "missing").is_none()); + } + + #[test] + fn array_field_is_empty_cases() { + assert!(array_field_is_empty(&v(json!({"a": []})), "a")); + assert!(!array_field_is_empty(&v(json!({"a": [1]})), "a")); + // Absent field, and a present-but-not-array field, both count as empty. + assert!(array_field_is_empty(&v(json!({})), "a")); + assert!(array_field_is_empty(&v(json!({"a": "x"})), "a")); + } +} diff --git a/src/transforms/policy/otlp_log_adapter.rs b/src/transforms/policy/otlp_log_adapter.rs new file mode 100644 index 0000000000000..9915eb2b71276 --- /dev/null +++ b/src/transforms/policy/otlp_log_adapter.rs @@ -0,0 +1,753 @@ +//! OTLP envelope adapter for the `policy` transform. +//! +//! When the transform runs in `mode: otel`, each incoming Vector event is +//! treated as an OTLP envelope (`{ resourceLogs: [...] }`) emitted by +//! Vector's `opentelemetry` source with `use_otlp_decoding.logs = true`. +//! This module iterates every `logRecord` inside the envelope, evaluates +//! it through `policy-rs` via [`OtlpLogAdapter`], filters in place, and +//! prunes empty `scopeLogs` / `resourceLogs` entries. +//! +//! The adapter mirrors the conformance reference adapter +//! (`policy-conformance/runners/rs/src/eval.rs`) so behaviour matches the +//! spec exactly: +//! +//! * Field names are camelCase per the proto3 JSON mapping +//! (`severityText`, not `severity_text`). +//! * `body` is an OTLP `AnyValue`; only a non-empty `stringValue` is +//! matchable. Non-string variants (`intValue`, `boolValue`, …) are not +//! coerced to strings for matching, so a regex redact targeting a string +//! never mutates them. +//! * Simple string fields (`severityText`, `traceId`, `spanId`, +//! `eventName`, schema URLs) treat the empty string as absent. +//! * `traceId` / `spanId` arrive already hex-encoded (the OTLP decoder +//! normalizes the raw `bytes` to canonical OTLP/JSON hex), so they are +//! plain strings here. +//! * Attributes are arrays of `{ key, value: AnyValue }`. Multi-segment +//! selectors (`["http", "method"]`) walk nested `kvlistValue` entries. +//! * Resource and scope attributes are mutable: the envelope iteration +//! lends the adapter `&mut` access to the parent `resource` / `scope` +//! objects, so add / redact / remove / rename on +//! `ResourceAttribute` / `ScopeAttribute` selectors mutate them in place. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::LogField; +use policy_rs::{ + EvaluateResult, LogFieldSelector, Matchable, PolicyEngine, PolicySnapshot, Transformable, + engine::LogSignal, +}; +use vector_lib::event::{LogEvent, ObjectMap, Value}; + +use super::internal_events::{DropCounts, DropReason, EvalErrors}; +use super::otlp_common::{ + any_value_string, attribute_exists_path, attribute_key_eq, find_attribute_path, lift_child, + non_empty, reattach_child, +}; + +/// Iterate every record inside an OTLP envelope event, applying policies +/// per-record. Returns `Some(log)` to forward (with mutated and possibly +/// pruned contents), or `None` if every record was filtered out and the +/// envelope should be dropped entirely. +/// +/// If the event is not envelope-shaped (no `resourceLogs` key, or it's +/// not an array), the event is forwarded unchanged so users who +/// accidentally send non-envelope events through an `otel`-mode transform +/// don't lose data silently. +pub(super) async fn evaluate_logs_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mut log: LogEvent, +) -> Option { + let Some(resource_logs) = log.get_mut("resourceLogs").and_then(Value::as_array_mut) else { + // Not an envelope. Pass through. + return Some(log); + }; + + let mut drops = DropCounts::default(); + let mut errors = EvalErrors::default(); + let mut i = 0; + while i < resource_logs.len() { + // Lift `resource`/`scope` out (a move, not a clone) so the adapter can + // mutate them alongside the mutable borrow of the records array (the + // borrow checker can't prove those sibling paths are disjoint). They + // are re-attached after the records under them are processed. + let mut resource = lift_child(&mut resource_logs[i], "resource"); + let resource_schema_url = resource_logs[i].get("schemaUrl").cloned(); + + let mut prune_this_rl = false; + + if let Some(scope_logs) = resource_logs[i] + .get_mut("scopeLogs") + .and_then(Value::as_array_mut) + { + let mut j = 0; + while j < scope_logs.len() { + let mut scope = lift_child(&mut scope_logs[j], "scope"); + let scope_schema_url = scope_logs[j].get("schemaUrl").cloned(); + + let mut prune_this_sl = false; + + if let Some(records) = scope_logs[j] + .get_mut("logRecords") + .and_then(Value::as_array_mut) + { + let mut k = 0; + while k < records.len() { + let result = { + let mut adapter = OtlpLogAdapter { + log_record: &mut records[k], + resource: resource.as_mut(), + scope: scope.as_mut(), + resource_schema_url: resource_schema_url.as_ref(), + scope_schema_url: scope_schema_url.as_ref(), + }; + engine.evaluate_and_transform(snapshot, &mut adapter).await + }; + + match result { + Ok(EvaluateResult::NoMatch) + | Ok(EvaluateResult::Keep { .. }) + | Ok(EvaluateResult::Sample { keep: true, .. }) + | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => { + k += 1; + } + Ok(EvaluateResult::Drop { .. }) => { + records.remove(k); + drops.record(DropReason::PolicyDrop); + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + records.remove(k); + drops.record(DropReason::SampleRejected); + } + Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { + records.remove(k); + drops.record(DropReason::RateLimited); + } + Err(error) => { + errors.record(&error); + k += 1; + } + } + } + prune_this_sl = records.is_empty(); + } + + // Re-attach scope only if the entry survives (otherwise it's + // about to be pruned, so re-inserting would be wasted work). + if !prune_this_sl && let Some(scope) = scope { + reattach_child(&mut scope_logs[j], "scope", scope); + } + + if prune_this_sl { + scope_logs.remove(j); + } else { + j += 1; + } + } + prune_this_rl = scope_logs.is_empty(); + } + + if !prune_this_rl && let Some(resource) = resource { + reattach_child(&mut resource_logs[i], "resource", resource); + } + + if prune_this_rl { + resource_logs.remove(i); + } else { + i += 1; + } + } + + drops.emit(); + errors.emit(); + + if resource_logs.is_empty() { + None + } else { + Some(log) + } +} + +/// Adapter exposing a single OTLP `logRecord` (plus its parent resource +/// and scope) to the `policy-rs` engine. +pub(super) struct OtlpLogAdapter<'a> { + log_record: &'a mut Value, + resource: Option<&'a mut Value>, + scope: Option<&'a mut Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl<'a> OtlpLogAdapter<'a> { + /// Test-only positional constructor. Production code builds the adapter + /// with a struct literal (see `evaluate_logs_envelope`) so the four same-typed + /// optional borrows can't be transposed silently. + #[cfg(test)] + pub(super) fn new( + log_record: &'a mut Value, + resource: Option<&'a mut Value>, + scope: Option<&'a mut Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, + ) -> Self { + Self { + log_record, + resource, + scope, + resource_schema_url, + scope_schema_url, + } + } + + /// Borrow the attributes array for an attribute-namespace selector. + fn attributes_for(&self, selector: &LogFieldSelector) -> Option<&Value> { + match selector { + LogFieldSelector::LogAttribute(_) => self.log_record.get("attributes"), + LogFieldSelector::ResourceAttribute(_) => { + self.resource.as_deref().and_then(|r| r.get("attributes")) + } + LogFieldSelector::ScopeAttribute(_) => { + self.scope.as_deref().and_then(|s| s.get("attributes")) + } + LogFieldSelector::Simple(_) => None, + } + } +} + +impl Matchable for OtlpLogAdapter<'_> { + type Signal = LogSignal; + + fn get_field(&self, field: &LogFieldSelector) -> Option> { + match field { + LogFieldSelector::Simple(LogField::Body) => { + any_value_string(self.log_record.get("body")) + } + LogFieldSelector::Simple(LogField::SeverityText) => { + non_empty(self.log_record.get("severityText")) + } + LogFieldSelector::Simple(LogField::TraceId) => { + non_empty(self.log_record.get("traceId")) + } + LogFieldSelector::Simple(LogField::SpanId) => non_empty(self.log_record.get("spanId")), + LogFieldSelector::Simple(LogField::EventName) => { + non_empty(self.log_record.get("eventName")) + } + LogFieldSelector::Simple(LogField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + LogFieldSelector::Simple(LogField::ScopeSchemaUrl) => non_empty(self.scope_schema_url), + LogFieldSelector::Simple(LogField::Unspecified) => None, + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => { + find_attribute_path(self.attributes_for(field), path) + } + } + } + + fn field_exists(&self, field: &LogFieldSelector) -> bool { + match field { + LogFieldSelector::Simple(LogField::Body) => { + log_body_present(self.log_record.get("body")) + } + LogFieldSelector::Simple(LogField::Unspecified) => false, + LogFieldSelector::Simple(_) => self.get_field(field).is_some(), + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.attributes_for(field), path) + } + } + } +} + +impl Transformable for OtlpLogAdapter<'_> { + fn set_field(&mut self, field: &LogFieldSelector, value: &str) { + match field { + LogFieldSelector::Simple(LogField::Body) => { + if let Some(obj) = self.log_record.as_object_mut() { + obj.insert("body".into(), make_string_any_value(value)); + } + } + LogFieldSelector::Simple(LogField::SeverityText) => { + insert_simple_string(self.log_record, "severityText", value); + } + LogFieldSelector::Simple(LogField::TraceId) => { + insert_simple_string(self.log_record, "traceId", value); + } + LogFieldSelector::Simple(LogField::SpanId) => { + insert_simple_string(self.log_record, "spanId", value); + } + LogFieldSelector::Simple(LogField::EventName) => { + insert_simple_string(self.log_record, "eventName", value); + } + // Schema URLs / unspecified have no log-record-local storage. + LogFieldSelector::Simple(_) => {} + LogFieldSelector::LogAttribute(path) => { + if let Some(attrs) = ensure_attributes(self.log_record) { + set_string_attr(attrs, path, value); + } + } + LogFieldSelector::ResourceAttribute(path) => { + if let Some(resource) = self.resource.as_deref_mut() + && let Some(attrs) = ensure_attributes(resource) + { + set_string_attr(attrs, path, value); + } + } + LogFieldSelector::ScopeAttribute(path) => { + if let Some(scope) = self.scope.as_deref_mut() + && let Some(attrs) = ensure_attributes(scope) + { + set_string_attr(attrs, path, value); + } + } + } + } + + fn delete_field(&mut self, field: &LogFieldSelector) -> bool { + match field { + LogFieldSelector::Simple(LogField::Body) => remove_key(self.log_record, "body"), + LogFieldSelector::Simple(LogField::SeverityText) => { + remove_key(self.log_record, "severityText") + } + LogFieldSelector::Simple(LogField::TraceId) => remove_key(self.log_record, "traceId"), + LogFieldSelector::Simple(LogField::SpanId) => remove_key(self.log_record, "spanId"), + LogFieldSelector::Simple(LogField::EventName) => { + remove_key(self.log_record, "eventName") + } + LogFieldSelector::Simple(_) => false, + LogFieldSelector::LogAttribute(path) => attributes_of_mut(self.log_record) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), + LogFieldSelector::ResourceAttribute(path) => self + .resource + .as_deref_mut() + .and_then(attributes_of_mut) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), + LogFieldSelector::ScopeAttribute(path) => self + .scope + .as_deref_mut() + .and_then(attributes_of_mut) + .map(|attrs| remove_attr(attrs, path)) + .unwrap_or(false), + } + } + + fn move_field(&mut self, from: &LogFieldSelector, to: &LogFieldSelector) { + // The engine guarantees `from` exists. Remove the underlying + // `{key, value}` entry (preserving the OTel value type) and re-insert + // it under `to`'s key in `to`'s namespace, overwriting any existing + // entry there (upsert semantics, matching the reference adapter). + let source = match from { + LogFieldSelector::LogAttribute(path) => { + attributes_of_mut(self.log_record).and_then(|attrs| remove_attr_kv(attrs, path)) + } + LogFieldSelector::ResourceAttribute(path) => self + .resource + .as_deref_mut() + .and_then(attributes_of_mut) + .and_then(|attrs| remove_attr_kv(attrs, path)), + LogFieldSelector::ScopeAttribute(path) => self + .scope + .as_deref_mut() + .and_then(attributes_of_mut) + .and_then(|attrs| remove_attr_kv(attrs, path)), + LogFieldSelector::Simple(_) => None, + }; + let Some(mut entry) = source else { + return; + }; + + let target_key = match to { + LogFieldSelector::LogAttribute(path) + | LogFieldSelector::ResourceAttribute(path) + | LogFieldSelector::ScopeAttribute(path) => path.first().cloned(), + LogFieldSelector::Simple(_) => None, + }; + let Some(key) = target_key else { + return; + }; + if let Some(obj) = entry.as_object_mut() { + obj.insert("key".into(), Value::from(key.clone())); + } + + match to { + LogFieldSelector::LogAttribute(_) => { + if let Some(attrs) = ensure_attributes(self.log_record) { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::ResourceAttribute(_) => { + if let Some(resource) = self.resource.as_deref_mut() + && let Some(attrs) = ensure_attributes(resource) + { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::ScopeAttribute(_) => { + if let Some(scope) = self.scope.as_deref_mut() + && let Some(attrs) = ensure_attributes(scope) + { + upsert_entry(attrs, &key, entry); + } + } + LogFieldSelector::Simple(_) => {} + } + } +} + +// ============================================================================= +// Log-record `body` presence (log-specific `AnyValue` semantics). +// ============================================================================= + +/// Presence semantics for the `body` field: an empty `stringValue` counts as +/// missing, but any other present variant counts as present. +fn log_body_present(value: Option<&Value>) -> bool { + let Some(obj) = value.and_then(Value::as_object) else { + return false; + }; + if let Some(s) = obj.get("stringValue").and_then(Value::as_str) { + return !s.is_empty(); + } + [ + "boolValue", + "intValue", + "doubleValue", + "arrayValue", + "kvlistValue", + "bytesValue", + ] + .iter() + .any(|k| obj.get(*k).is_some()) +} + +// ============================================================================= +// Attribute mutation. +// ============================================================================= + +/// Set or overwrite the first-segment attribute key with a string value. +fn set_string_attr(attrs: &mut Vec, path: &[String], value: &str) { + let Some(key) = path.first() else { + return; + }; + if let Some(entry) = attrs.iter_mut().find(|kv| attribute_key_eq(kv, key)) { + if let Some(obj) = entry.as_object_mut() { + obj.insert("value".into(), make_string_any_value(value)); + } + return; + } + attrs.push(make_attribute_entry(key, make_string_any_value(value))); +} + +/// Remove the first-segment attribute key. Returns whether anything was removed. +fn remove_attr(attrs: &mut Vec, path: &[String]) -> bool { + let Some(key) = path.first() else { + return false; + }; + let before = attrs.len(); + attrs.retain(|kv| !attribute_key_eq(kv, key)); + attrs.len() < before +} + +/// Remove and return the first-segment attribute entry, preserving its value. +fn remove_attr_kv(attrs: &mut Vec, path: &[String]) -> Option { + let key = path.first()?; + let idx = attrs.iter().position(|kv| attribute_key_eq(kv, key))?; + Some(attrs.remove(idx)) +} + +/// Insert an entry under `key`, removing any existing entry with that key first. +fn upsert_entry(attrs: &mut Vec, key: &str, entry: Value) { + attrs.retain(|kv| !attribute_key_eq(kv, key)); + attrs.push(entry); +} + +// ============================================================================= +// Value construction. +// ============================================================================= + +/// Build an `AnyValue` object wrapping a string. +fn make_string_any_value(value: &str) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("stringValue".into(), Value::from(value.to_string())); + Value::Object(obj) +} + +/// Build a single OTLP attribute entry `{ key, value }`. +fn make_attribute_entry(key: &str, value: Value) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("key".into(), Value::from(key.to_string())); + obj.insert("value".into(), value); + Value::Object(obj) +} + +/// Get-or-create the `attributes` array of a record / resource / scope. +fn ensure_attributes(parent: &mut Value) -> Option<&mut Vec> { + let obj = parent.as_object_mut()?; + if !matches!(obj.get("attributes"), Some(Value::Array(_))) { + obj.insert("attributes".into(), Value::Array(Vec::new())); + } + obj.get_mut("attributes").and_then(Value::as_array_mut) +} + +/// Borrow the existing `attributes` array, if present. +fn attributes_of_mut(parent: &mut Value) -> Option<&mut Vec> { + parent + .as_object_mut() + .and_then(|o| o.get_mut("attributes")) + .and_then(Value::as_array_mut) +} + +fn remove_key(record: &mut Value, key: &str) -> bool { + match record.as_object_mut() { + Some(obj) => obj.remove(key).is_some(), + None => false, + } +} + +fn insert_simple_string(record: &mut Value, key: &str, value: &str) { + if let Some(obj) = record.as_object_mut() { + obj.insert(key.into(), Value::from(value.to_string())); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn any_int(v: i64) -> Value { + let mut obj = ObjectMap::new(); + obj.insert("intValue".into(), Value::Integer(v)); + Value::Object(obj) + } + + fn record_with_attr(attr_key: &str, attr_val: Value) -> Value { + let mut record = ObjectMap::new(); + record.insert("body".into(), make_string_any_value("hi")); + record.insert("severityText".into(), Value::from("INFO".to_string())); + record.insert( + "attributes".into(), + Value::Array(vec![make_attribute_entry(attr_key, attr_val)]), + ); + Value::Object(record) + } + + fn obj_with_attrs(pairs: Vec<(&str, Value)>) -> Value { + let mut o = ObjectMap::new(); + o.insert( + "attributes".into(), + Value::Array( + pairs + .into_iter() + .map(|(k, v)| make_attribute_entry(k, v)) + .collect(), + ), + ); + Value::Object(o) + } + + fn get(adapter: &OtlpLogAdapter, sel: LogFieldSelector) -> Option { + adapter.get_field(&sel).map(|c| c.into_owned()) + } + + #[test] + fn matches_string_attribute() { + let mut record = record_with_attr("user_id", make_string_any_value("42")); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["user_id".into()]) + ), + Some("42".to_string()) + ); + } + + #[test] + fn int_attribute_is_not_matchable_but_exists() { + let mut record = record_with_attr("count", any_int(42)); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + // get_field must return None (not "42") so a regex redact won't fire. + assert_eq!( + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["count".into()]) + ), + None + ); + // but exists: true must still match. + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec!["count".into()]))); + } + + #[test] + fn empty_string_simple_field_is_absent() { + let mut record = ObjectMap::new(); + record.insert("spanId".into(), Value::from(String::new())); + record.insert("severityText".into(), Value::from("INFO".to_string())); + let mut record = Value::Object(record); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::SpanId))); + assert!(adapter.field_exists(&LogFieldSelector::Simple(LogField::SeverityText))); + } + + #[test] + fn hex_span_id_is_matched_verbatim() { + let mut record = ObjectMap::new(); + record.insert("spanId".into(), Value::from("7370616e30303031".to_string())); + let mut record = Value::Object(record); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + get(&adapter, LogFieldSelector::Simple(LogField::SpanId)), + Some("7370616e30303031".to_string()) + ); + } + + #[test] + fn body_present_only_when_non_empty() { + // Missing body. + let mut record = Value::Object(ObjectMap::new()); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + + // Empty AnyValue (body: {}) counts as missing. + let mut record = ObjectMap::new(); + record.insert("body".into(), Value::Object(ObjectMap::new())); + let mut record = Value::Object(record); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert!(!adapter.field_exists(&LogFieldSelector::Simple(LogField::Body))); + } + + #[test] + fn add_body_when_missing() { + let mut record = Value::Object(ObjectMap::new()); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + adapter.set_field( + &LogFieldSelector::Simple(LogField::Body), + "[no body provided]", + ); + } + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + get(&adapter, LogFieldSelector::Simple(LogField::Body)), + Some("[no body provided]".to_string()) + ); + } + + #[test] + fn walks_nested_kvlist() { + let mut inner = ObjectMap::new(); + inner.insert( + "values".into(), + Value::Array(vec![make_attribute_entry( + "method", + make_string_any_value("GET"), + )]), + ); + let mut http_val = ObjectMap::new(); + http_val.insert("kvlistValue".into(), Value::Object(inner)); + let mut record = record_with_attr("http", Value::Object(http_val)); + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + assert_eq!( + get( + &adapter, + LogFieldSelector::LogAttribute(vec!["http".into(), "method".into()]) + ), + Some("GET".to_string()) + ); + assert!(adapter.field_exists(&LogFieldSelector::LogAttribute(vec![ + "http".into(), + "method".into() + ]))); + } + + #[test] + fn resource_attribute_is_mutable() { + let mut record = record_with_attr("noise", make_string_any_value("v")); + let mut resource = obj_with_attrs(vec![("existing", make_string_any_value("x"))]); + { + let mut adapter = + OtlpLogAdapter::new(&mut record, Some(&mut resource), None, None, None); + adapter.set_field( + &LogFieldSelector::ResourceAttribute(vec!["processed_by".into()]), + "policy", + ); + } + let adapter = OtlpLogAdapter::new(&mut record, Some(&mut resource), None, None, None); + assert_eq!( + get( + &adapter, + LogFieldSelector::ResourceAttribute(vec!["processed_by".into()]) + ), + Some("policy".to_string()) + ); + } + + #[test] + fn scope_attribute_remove_and_rename() { + let mut record = record_with_attr("noise", make_string_any_value("v")); + let mut scope = obj_with_attrs(vec![ + ("secret", make_string_any_value("abc")), + ("old", make_string_any_value("val")), + ]); + { + let mut adapter = OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); + assert!(adapter.delete_field(&LogFieldSelector::ScopeAttribute(vec!["secret".into()]))); + adapter.move_field( + &LogFieldSelector::ScopeAttribute(vec!["old".into()]), + &LogFieldSelector::ScopeAttribute(vec!["new".into()]), + ); + } + let adapter = OtlpLogAdapter::new(&mut record, None, Some(&mut scope), None, None); + assert!( + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["secret".into()]) + ) + .is_none() + ); + assert!( + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["old".into()]) + ) + .is_none() + ); + assert_eq!( + get( + &adapter, + LogFieldSelector::ScopeAttribute(vec!["new".into()]) + ), + Some("val".to_string()) + ); + } + + #[test] + fn redact_regex_skips_non_string_value() { + // Mirrors logs_transform_redact_regex_non_string_value: an intValue + // attribute must be left untouched because get_field returns None. + let mut record = record_with_attr("count", any_int(42)); + { + let adapter = OtlpLogAdapter::new(&mut record, None, None, None, None); + // The engine reads the value first; None means redact is skipped. + assert!( + adapter + .get_field(&LogFieldSelector::LogAttribute(vec!["count".into()])) + .is_none() + ); + } + // The underlying intValue is still present and unchanged. + let count = record + .get("attributes") + .and_then(Value::as_array) + .and_then(|a| a.first()) + .and_then(|kv| kv.as_object()) + .and_then(|o| o.get("value")) + .and_then(Value::as_object) + .and_then(|o| o.get("intValue")) + .cloned(); + assert_eq!(count, Some(Value::Integer(42))); + } +} diff --git a/src/transforms/policy/otlp_metric_adapter.rs b/src/transforms/policy/otlp_metric_adapter.rs new file mode 100644 index 0000000000000..037418c16e163 --- /dev/null +++ b/src/transforms/policy/otlp_metric_adapter.rs @@ -0,0 +1,398 @@ +//! OTLP metrics adapter for the `policy` transform (`mode: otel`). +//! +//! When Vector's `opentelemetry` source decodes OTLP metrics with +//! `use_otlp_decoding`, the metrics arrive as a `Log` event shaped like +//! `{ resourceMetrics: [...] }`. This module iterates every metric inside the +//! envelope, evaluates it through `policy-rs` (match-only — metrics are not +//! transformed), drops metrics whose winning policy says so, and prunes empty +//! `scopeMetrics` / `resourceMetrics`. +//! +//! Mirrors the conformance reference (`MetricContext` in +//! `policy-conformance/runners/rs/src/eval.rs`): a metric is matched on the +//! attributes of its **first** data point, its descriptor fields, its type +//! (gauge/sum/…) and aggregation temporality. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::MetricField; +use policy_rs::{ + EvaluateResult, Matchable, MetricFieldSelector, PolicyEngine, PolicySnapshot, + engine::MetricSignal, +}; +use vector_lib::event::{LogEvent, Value}; + +use super::internal_events::{DropReason, emit_dropped}; +use super::otlp_common::{ + array_field_is_empty, attribute_exists_path, find_attribute_path, non_empty, +}; + +/// Data-variant keys of an OTLP `Metric` (proto3 JSON), paired with the +/// `MetricFieldSelector::Type` string the engine matches against. +const METRIC_TYPES: [(&str, &str); 5] = [ + ("gauge", "METRIC_TYPE_GAUGE"), + ("sum", "METRIC_TYPE_SUM"), + ("histogram", "METRIC_TYPE_HISTOGRAM"), + ("exponentialHistogram", "METRIC_TYPE_EXPONENTIAL_HISTOGRAM"), + ("summary", "METRIC_TYPE_SUMMARY"), +]; + +/// Iterate every metric in an OTLP metrics envelope, dropping filtered ones. +/// Returns `Some(log)` to forward or `None` to drop the whole event. +pub(super) async fn evaluate_metrics_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mut log: LogEvent, +) -> Option { + let Some(resource_metrics) = log.get_mut("resourceMetrics").and_then(Value::as_array_mut) + else { + return Some(log); + }; + + let mut dropped = 0u64; + let mut i = 0; + while i < resource_metrics.len() { + // Phase 1 — score every metric. Metric evaluation is filter-only (no + // transform), so resource/scope/metric are all read-only here: we can + // borrow them immutably and avoid both the deep clone and the + // remove/re-insert lift the log/trace paths need. `keep[s][m]` is + // whether scope `s`'s metric `m` survives. + let keep: Vec> = { + let rm = &resource_metrics[i]; + let resource = rm.get("resource"); + let resource_schema_url = rm.get("schemaUrl"); + let mut per_scope = Vec::new(); + if let Some(scope_metrics) = rm.get("scopeMetrics").and_then(Value::as_array) { + for sm in scope_metrics { + let scope = sm.get("scope"); + let scope_schema_url = sm.get("schemaUrl"); + let mut per_metric = Vec::new(); + if let Some(metrics) = sm.get("metrics").and_then(Value::as_array) { + for metric in metrics { + let adapter = MetricAdapter { + metric, + resource, + scope, + resource_schema_url, + scope_schema_url, + }; + let drop = matches!( + engine.evaluate(snapshot, &adapter).await, + Ok(EvaluateResult::Drop { .. }) + ); + if drop { + dropped += 1; + } + per_metric.push(!drop); + } + } + per_scope.push(per_metric); + } + } + per_scope + }; + + // Phase 2 — apply the decisions, then prune emptied scopes / resource. + if let Some(scope_metrics) = resource_metrics[i] + .get_mut("scopeMetrics") + .and_then(Value::as_array_mut) + { + // Nothing mutates `scopeMetrics`/`metrics` between phases, so the + // decision arrays line up with the live arrays. Index defensively + // anyway (`.get(..)` + fail-open fallback) so a future desync + // degrades to "keep" instead of panicking the transform task. + debug_assert_eq!(keep.len(), scope_metrics.len()); + let mut scope_idx = 0; + let mut j = 0; + while j < scope_metrics.len() { + let scope_keep: &[bool] = keep.get(scope_idx).map_or(&[], Vec::as_slice); + scope_idx += 1; + if let Some(metrics) = scope_metrics[j] + .get_mut("metrics") + .and_then(Value::as_array_mut) + { + let mut m = 0; + metrics.retain(|_| { + let k = scope_keep.get(m).copied().unwrap_or(true); + m += 1; + k + }); + if metrics.is_empty() { + scope_metrics.remove(j); + continue; + } + } + j += 1; + } + } + + let prune_rm = array_field_is_empty(&resource_metrics[i], "scopeMetrics"); + if prune_rm { + resource_metrics.remove(i); + } else { + i += 1; + } + } + + emit_dropped(DropReason::PolicyDrop, dropped); + + if resource_metrics.is_empty() { + None + } else { + Some(log) + } +} + +/// Adapter exposing a single OTLP `metric` (plus parent resource/scope) to the +/// `policy-rs` engine. Read-only: metrics are filtered, never transformed. +struct MetricAdapter<'a> { + metric: &'a Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl MetricAdapter<'_> { + /// The data variant object (gauge/sum/…) and its canonical type string. + fn data(&self) -> Option<(&Value, &'static str)> { + let obj = self.metric.as_object()?; + METRIC_TYPES + .iter() + .find_map(|(key, ty)| obj.get(*key).map(|v| (v, *ty))) + } + + /// Attributes array of the first data point, if any. + fn first_datapoint_attributes(&self) -> Option<&Value> { + let (data, _) = self.data()?; + data.as_object() + .and_then(|o| o.get("dataPoints")) + .and_then(Value::as_array) + .and_then(|points| points.first()) + .and_then(|p| p.as_object()) + .and_then(|o| o.get("attributes")) + } + + fn resource_attributes(&self) -> Option<&Value> { + self.resource.and_then(|r| r.get("attributes")) + } + + fn scope_attributes(&self) -> Option<&Value> { + self.scope.and_then(|s| s.get("attributes")) + } +} + +impl Matchable for MetricAdapter<'_> { + type Signal = MetricSignal; + + fn get_field(&self, field: &MetricFieldSelector) -> Option> { + match field { + MetricFieldSelector::Simple(MetricField::Name) => non_empty(self.metric.get("name")), + MetricFieldSelector::Simple(MetricField::Description) => { + non_empty(self.metric.get("description")) + } + MetricFieldSelector::Simple(MetricField::Unit) => non_empty(self.metric.get("unit")), + MetricFieldSelector::Simple(MetricField::ScopeName) => { + non_empty(self.scope.and_then(|s| s.get("name"))) + } + MetricFieldSelector::Simple(MetricField::ScopeVersion) => { + non_empty(self.scope.and_then(|s| s.get("version"))) + } + MetricFieldSelector::Simple(MetricField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + MetricFieldSelector::Simple(MetricField::ScopeSchemaUrl) => { + non_empty(self.scope_schema_url) + } + MetricFieldSelector::Simple(MetricField::Unspecified) => None, + MetricFieldSelector::DatapointAttribute(path) => { + find_attribute_path(self.first_datapoint_attributes(), path) + } + MetricFieldSelector::ResourceAttribute(path) => { + find_attribute_path(self.resource_attributes(), path) + } + MetricFieldSelector::ScopeAttribute(path) => { + find_attribute_path(self.scope_attributes(), path) + } + MetricFieldSelector::Type => self.data().map(|(_, ty)| Cow::Borrowed(ty)), + MetricFieldSelector::Temporality => { + let (data, _) = self.data()?; + non_empty( + data.as_object() + .and_then(|o| o.get("aggregationTemporality")), + ) + } + } + } + + fn field_exists(&self, field: &MetricFieldSelector) -> bool { + match field { + MetricFieldSelector::DatapointAttribute(path) => { + attribute_exists_path(self.first_datapoint_attributes(), path) + } + MetricFieldSelector::ResourceAttribute(path) => { + attribute_exists_path(self.resource_attributes(), path) + } + MetricFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.scope_attributes(), path) + } + _ => self.get_field(field).is_some(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn val(value: serde_json::Value) -> Value { + Value::from(value) + } + + fn adapter<'a>( + metric: &'a Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + ) -> MetricAdapter<'a> { + MetricAdapter { + metric, + resource, + scope, + resource_schema_url: None, + scope_schema_url: None, + } + } + + fn get(metric: &Value, sel: MetricFieldSelector) -> Option { + adapter(metric, None, None) + .get_field(&sel) + .map(Cow::into_owned) + } + + #[test] + fn metric_type_derived_from_data_variant() { + let cases = [ + ( + json!({"name": "m", "gauge": {"dataPoints": []}}), + "METRIC_TYPE_GAUGE", + ), + ( + json!({"name": "m", "sum": {"dataPoints": []}}), + "METRIC_TYPE_SUM", + ), + ( + json!({"name": "m", "histogram": {"dataPoints": []}}), + "METRIC_TYPE_HISTOGRAM", + ), + ( + json!({"name": "m", "exponentialHistogram": {"dataPoints": []}}), + "METRIC_TYPE_EXPONENTIAL_HISTOGRAM", + ), + ( + json!({"name": "m", "summary": {"dataPoints": []}}), + "METRIC_TYPE_SUMMARY", + ), + ]; + for (metric, expected) in cases { + let metric = val(metric); + assert_eq!( + get(&metric, MetricFieldSelector::Type), + Some(expected.to_string()), + ); + } + } + + #[test] + fn temporality_read_from_data_variant() { + let delta = val(json!({ + "name": "m", + "sum": {"dataPoints": [], "aggregationTemporality": "AGGREGATION_TEMPORALITY_DELTA"} + })); + assert_eq!( + get(&delta, MetricFieldSelector::Temporality), + Some("AGGREGATION_TEMPORALITY_DELTA".to_string()), + ); + + // Gauge has no temporality. + let gauge = val(json!({"name": "m", "gauge": {"dataPoints": []}})); + assert_eq!(get(&gauge, MetricFieldSelector::Temporality), None); + } + + #[test] + fn descriptor_fields() { + let m = val(json!({ + "name": "http.requests", "description": "count", "unit": "1", + "gauge": {"dataPoints": []} + })); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Name)), + Some("http.requests".to_string()), + ); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Description)), + Some("count".to_string()), + ); + assert_eq!( + get(&m, MetricFieldSelector::Simple(MetricField::Unit)), + Some("1".to_string()), + ); + } + + #[test] + fn datapoint_attribute_uses_first_datapoint() { + let m = val(json!({"name": "m", "sum": {"dataPoints": [ + {"attributes": [{"key": "http.method", "value": {"stringValue": "GET"}}]}, + {"attributes": [{"key": "http.method", "value": {"stringValue": "POST"}}]} + ]}})); + assert_eq!( + get( + &m, + MetricFieldSelector::DatapointAttribute(vec!["http.method".to_string()]) + ), + Some("GET".to_string()), + ); + } + + #[test] + fn non_string_datapoint_attr_exists_but_is_unmatchable() { + let m = val(json!({"name": "m", "sum": {"dataPoints": [ + {"attributes": [{"key": "count", "value": {"intValue": "42"}}]} + ]}})); + let sel = MetricFieldSelector::DatapointAttribute(vec!["count".to_string()]); + // Not coercible to a string for matching... + assert_eq!(get(&m, sel.clone()), None); + // ...but `exists` still fires. + assert!(adapter(&m, None, None).field_exists(&sel)); + } + + #[test] + fn resource_and_scope_fields() { + let m = val(json!({"name": "m", "gauge": {"dataPoints": []}})); + let resource = + val(json!({"attributes": [{"key": "service.name", "value": {"stringValue": "api"}}]})); + let scope = val(json!({ + "name": "lib", "version": "1.2", + "attributes": [{"key": "k", "value": {"stringValue": "x"}}] + })); + let adapter = adapter(&m, Some(&resource), Some(&scope)); + let g = |sel| adapter.get_field(&sel).map(Cow::into_owned); + assert_eq!( + g(MetricFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])), + Some("api".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::ScopeAttribute(vec!["k".to_string()])), + Some("x".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::Simple(MetricField::ScopeName)), + Some("lib".to_string()), + ); + assert_eq!( + g(MetricFieldSelector::Simple(MetricField::ScopeVersion)), + Some("1.2".to_string()), + ); + } +} diff --git a/src/transforms/policy/otlp_trace_adapter.rs b/src/transforms/policy/otlp_trace_adapter.rs new file mode 100644 index 0000000000000..cfd4168b0ead9 --- /dev/null +++ b/src/transforms/policy/otlp_trace_adapter.rs @@ -0,0 +1,545 @@ +//! OTLP traces adapter for the `policy` transform (`mode: otel`). +//! +//! Vector's `opentelemetry` source decodes OTLP traces into a `Trace` event +//! shaped like `{ resourceSpans: [...] }`. This module iterates every span, +//! evaluates it through `policy-rs` (`evaluate_trace`, which applies OTel +//! consistent-probability sampling), drops filtered spans, writes the sampling +//! threshold back into the span's `traceState`, and prunes empty +//! `scopeSpans` / `resourceSpans`. +//! +//! Mirrors the conformance reference (`MutTraceContext` in +//! `policy-conformance/runners/rs/src/eval.rs`). Resource and scope are +//! read-only in trace mode; only the span (its `traceState`) is mutated. + +use std::borrow::Cow; + +use policy_rs::proto::tero::policy::v1::TraceField; +use policy_rs::{ + EvaluateResult, Matchable, PolicyEngine, PolicySnapshot, TraceFieldSelector, Transformable, + engine::TraceSignal, +}; +use vector_lib::event::{TraceEvent, Value}; + +use super::internal_events::{DropCounts, DropReason, EvalErrors}; +use super::otlp_common::{ + attribute_exists_path, find_attribute_path, lift_child, non_empty, reattach_child, +}; + +/// Iterate every span in an OTLP traces envelope, sampling/dropping in place. +/// Returns `true` if any span survives (forward the event), `false` if the +/// envelope is now empty and the event should be dropped. +pub(super) async fn evaluate_traces_envelope( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + trace: &mut TraceEvent, +) -> bool { + let Some(resource_spans) = trace + .value_mut() + .get_mut("resourceSpans") + .and_then(Value::as_array_mut) + else { + // Not an envelope. Forward unchanged. + return true; + }; + + let mut drops = DropCounts::default(); + let mut errors = EvalErrors::default(); + let mut i = 0; + while i < resource_spans.len() { + // Lift resource/scope out (a move, not a clone) so the adapter can read + // them while we mutate the sibling `spans` array; re-attached after. + let resource = lift_child(&mut resource_spans[i], "resource"); + let resource_schema_url = resource_spans[i].get("schemaUrl").cloned(); + + let mut prune_rs = false; + + if let Some(scope_spans) = resource_spans[i] + .get_mut("scopeSpans") + .and_then(Value::as_array_mut) + { + let mut j = 0; + while j < scope_spans.len() { + let scope = lift_child(&mut scope_spans[j], "scope"); + let scope_schema_url = scope_spans[j].get("schemaUrl").cloned(); + + let mut prune_ss = false; + + if let Some(spans) = scope_spans[j] + .get_mut("spans") + .and_then(Value::as_array_mut) + { + let mut k = 0; + while k < spans.len() { + let result = { + let mut adapter = TraceAdapter { + span: &mut spans[k], + resource: resource.as_ref(), + scope: scope.as_ref(), + resource_schema_url: resource_schema_url.as_ref(), + scope_schema_url: scope_schema_url.as_ref(), + }; + engine.evaluate_trace(snapshot, &mut adapter).await + }; + + let keep = match result { + Ok(EvaluateResult::Drop { .. }) => { + drops.record(DropReason::PolicyDrop); + false + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + drops.record(DropReason::SampleRejected); + false + } + Ok(_) => true, + Err(error) => { + errors.record(&error); + true + } + }; + + if keep { + k += 1; + } else { + spans.remove(k); + } + } + prune_ss = spans.is_empty(); + } + + // Re-attach scope only if the entry survives. + if !prune_ss && let Some(scope) = scope { + reattach_child(&mut scope_spans[j], "scope", scope); + } + + if prune_ss { + scope_spans.remove(j); + } else { + j += 1; + } + } + prune_rs = scope_spans.is_empty(); + } + + if !prune_rs && let Some(resource) = resource { + reattach_child(&mut resource_spans[i], "resource", resource); + } + + if prune_rs { + resource_spans.remove(i); + } else { + i += 1; + } + } + + drops.emit(); + errors.emit(); + !resource_spans.is_empty() +} + +/// Adapter exposing a single OTLP span (plus parent resource/scope) to the +/// `policy-rs` engine. The span is mutable so the engine can write the +/// sampling threshold into `traceState`; resource/scope are read-only. +struct TraceAdapter<'a> { + span: &'a mut Value, + resource: Option<&'a Value>, + scope: Option<&'a Value>, + resource_schema_url: Option<&'a Value>, + scope_schema_url: Option<&'a Value>, +} + +impl TraceAdapter<'_> { + fn span_attributes(&self) -> Option<&Value> { + self.span.get("attributes") + } + + fn resource_attributes(&self) -> Option<&Value> { + self.resource.and_then(|r| r.get("attributes")) + } + + fn scope_attributes(&self) -> Option<&Value> { + self.scope.and_then(|s| s.get("attributes")) + } + + /// First non-empty span event name, used by `EventName` matchers. + fn event_name(&self) -> Option> { + let events = self.span.get("events").and_then(Value::as_array)?; + for event in events { + if let Some(name) = non_empty(event.as_object().and_then(|o| o.get("name"))) { + return Some(name); + } + } + None + } + + /// Map the OTel `Status.code` enum to the policy `SPAN_STATUS_CODE_*` form. + /// + /// A span carries a status only if the `status` message is present. Its + /// `code` defaults to `STATUS_CODE_UNSET` (0), which proto3 omits on the + /// wire — so a present-but-empty `status` (or one whose `code` decoded to + /// the default) means UNSET, not "no status". A truly absent `status` + /// message yields `None`. + fn span_status(&self) -> Option> { + let status = self.span.get("status").and_then(Value::as_object)?; + match status.get("code").and_then(Value::as_str).as_deref() { + Some("STATUS_CODE_OK") => Some(Cow::Borrowed("SPAN_STATUS_CODE_OK")), + Some("STATUS_CODE_ERROR") => Some(Cow::Borrowed("SPAN_STATUS_CODE_ERROR")), + Some("STATUS_CODE_UNSET") | None => Some(Cow::Borrowed("SPAN_STATUS_CODE_UNSPECIFIED")), + Some(_) => None, + } + } +} + +impl Matchable for TraceAdapter<'_> { + type Signal = TraceSignal; + + fn get_field(&self, field: &TraceFieldSelector) -> Option> { + match field { + TraceFieldSelector::Simple(TraceField::Name) => non_empty(self.span.get("name")), + TraceFieldSelector::Simple(TraceField::TraceId) => non_empty(self.span.get("traceId")), + TraceFieldSelector::Simple(TraceField::SpanId) => non_empty(self.span.get("spanId")), + TraceFieldSelector::Simple(TraceField::ParentSpanId) => { + non_empty(self.span.get("parentSpanId")) + } + TraceFieldSelector::Simple(TraceField::TraceState) => { + non_empty(self.span.get("traceState")) + } + TraceFieldSelector::Simple(TraceField::ScopeName) => { + non_empty(self.scope.and_then(|s| s.get("name"))) + } + TraceFieldSelector::Simple(TraceField::ScopeVersion) => { + non_empty(self.scope.and_then(|s| s.get("version"))) + } + TraceFieldSelector::Simple(TraceField::ResourceSchemaUrl) => { + non_empty(self.resource_schema_url) + } + TraceFieldSelector::Simple(TraceField::ScopeSchemaUrl) => { + non_empty(self.scope_schema_url) + } + TraceFieldSelector::Simple(TraceField::Unspecified) => None, + TraceFieldSelector::SpanAttribute(path) => { + find_attribute_path(self.span_attributes(), path) + } + TraceFieldSelector::ResourceAttribute(path) => { + find_attribute_path(self.resource_attributes(), path) + } + TraceFieldSelector::ScopeAttribute(path) => { + find_attribute_path(self.scope_attributes(), path) + } + TraceFieldSelector::SpanKind => non_empty(self.span.get("kind")), + TraceFieldSelector::SpanStatus => self.span_status(), + TraceFieldSelector::EventName => self.event_name(), + // Not exercised by the conformance suite / not representable here. + TraceFieldSelector::EventAttribute(_) + | TraceFieldSelector::LinkTraceId + | TraceFieldSelector::SamplingThreshold => None, + } + } + + fn field_exists(&self, field: &TraceFieldSelector) -> bool { + match field { + TraceFieldSelector::SpanAttribute(path) => { + attribute_exists_path(self.span_attributes(), path) + } + TraceFieldSelector::ResourceAttribute(path) => { + attribute_exists_path(self.resource_attributes(), path) + } + TraceFieldSelector::ScopeAttribute(path) => { + attribute_exists_path(self.scope_attributes(), path) + } + _ => self.get_field(field).is_some(), + } + } +} + +impl Transformable for TraceAdapter<'_> { + fn set_field(&mut self, field: &TraceFieldSelector, value: &str) { + if matches!(field, TraceFieldSelector::SamplingThreshold) { + let current = self + .span + .get("traceState") + .and_then(Value::as_str) + .unwrap_or(Cow::Borrowed("")); + let merged = merge_ot_tracestate(¤t, &format!("th:{value}")); + if let Some(obj) = self.span.as_object_mut() { + obj.insert("traceState".into(), Value::from(merged)); + } + } + // Other trace transforms are not exercised by the conformance suite. + } + + fn delete_field(&mut self, _field: &TraceFieldSelector) -> bool { + false + } + + fn move_field(&mut self, _from: &TraceFieldSelector, _to: &TraceFieldSelector) {} +} + +/// Merge an OpenTelemetry sub-key (e.g. `"th:8000"`) into a W3C tracestate +/// string under the `ot` vendor key, replacing any existing entry with the +/// same sub-key. Ported verbatim from the conformance reference adapter. +fn merge_ot_tracestate(tracestate: &str, sub_kv: &str) -> String { + let sub_key = sub_kv.split(':').next().unwrap_or(sub_kv); + + let mut ot_parts: Vec<&str> = Vec::new(); + let mut other_vendors: Vec<&str> = Vec::new(); + + if !tracestate.is_empty() { + for vendor in tracestate.split(',') { + let vendor = vendor.trim(); + if vendor.is_empty() { + continue; + } + if let Some(ot_value) = vendor.strip_prefix("ot=") { + for part in ot_value.split(';') { + let part = part.trim(); + if part.is_empty() { + continue; + } + let part_key = part.split(':').next().unwrap_or(part); + if part_key != sub_key { + ot_parts.push(part); + } + } + } else { + other_vendors.push(vendor); + } + } + } + + let mut result = format!("ot={}", ot_parts.join(";")); + if !ot_parts.is_empty() { + result.push(';'); + } + result.push_str(sub_kv); + if !other_vendors.is_empty() { + result.push(','); + result.push_str(&other_vendors.join(",")); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn val(value: serde_json::Value) -> Value { + Value::from(value) + } + + /// Build a read-only adapter over an owned span value. + fn get(span: serde_json::Value, sel: TraceFieldSelector) -> Option { + let mut span = val(span); + let adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.get_field(&sel).map(Cow::into_owned) + } + + fn exists(span: serde_json::Value, sel: &TraceFieldSelector) -> bool { + let mut span = val(span); + let adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.field_exists(sel) + } + + #[test] + fn simple_span_fields() { + let span = json!({ + "name": "GET /x", "traceId": "abc", "spanId": "def", + "parentSpanId": "p", "traceState": "ot=th:8", "kind": "SPAN_KIND_SERVER" + }); + assert_eq!( + get(span.clone(), TraceFieldSelector::Simple(TraceField::Name)), + Some("GET /x".to_string()), + ); + assert_eq!( + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::TraceId) + ), + Some("abc".to_string()), + ); + assert_eq!( + get(span.clone(), TraceFieldSelector::Simple(TraceField::SpanId)), + Some("def".to_string()), + ); + assert_eq!( + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::ParentSpanId) + ), + Some("p".to_string()), + ); + assert_eq!( + get( + span.clone(), + TraceFieldSelector::Simple(TraceField::TraceState) + ), + Some("ot=th:8".to_string()), + ); + assert_eq!( + get(span, TraceFieldSelector::SpanKind), + Some("SPAN_KIND_SERVER".to_string()), + ); + } + + #[test] + fn span_status_maps_to_policy_codes() { + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_OK"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_OK".to_string()), + ); + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_ERROR"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_ERROR".to_string()), + ); + assert_eq!( + get( + json!({"status": {"code": "STATUS_CODE_UNSET"}}), + TraceFieldSelector::SpanStatus + ), + Some("SPAN_STATUS_CODE_UNSPECIFIED".to_string()), + ); + } + + #[test] + fn empty_status_is_unset_not_absent() { + // Regression guard: proto3 omits the default STATUS_CODE_UNSET, so a + // present-but-empty `status` must resolve to UNSPECIFIED and `exists`. + assert_eq!( + get(json!({"status": {}}), TraceFieldSelector::SpanStatus), + Some("SPAN_STATUS_CODE_UNSPECIFIED".to_string()), + ); + assert!(exists( + json!({"status": {}}), + &TraceFieldSelector::SpanStatus + )); + } + + #[test] + fn absent_status_is_none() { + assert_eq!( + get(json!({"name": "x"}), TraceFieldSelector::SpanStatus), + None, + ); + assert!(!exists( + json!({"name": "x"}), + &TraceFieldSelector::SpanStatus + )); + } + + #[test] + fn event_name_is_first_non_empty() { + assert_eq!( + get( + json!({"events": [{"name": ""}, {"name": "exception"}, {"name": "other"}]}), + TraceFieldSelector::EventName, + ), + Some("exception".to_string()), + ); + assert_eq!( + get(json!({"events": []}), TraceFieldSelector::EventName), + None, + ); + } + + #[test] + fn span_resource_scope_attributes() { + let mut span = + val(json!({"attributes": [{"key": "http.method", "value": {"stringValue": "GET"}}]})); + let resource = + val(json!({"attributes": [{"key": "service.name", "value": {"stringValue": "api"}}]})); + let scope = val(json!({"attributes": [{"key": "lib", "value": {"stringValue": "x"}}]})); + let adapter = TraceAdapter { + span: &mut span, + resource: Some(&resource), + scope: Some(&scope), + resource_schema_url: None, + scope_schema_url: None, + }; + let g = |sel| adapter.get_field(&sel).map(Cow::into_owned); + assert_eq!( + g(TraceFieldSelector::SpanAttribute(vec![ + "http.method".to_string() + ])), + Some("GET".to_string()), + ); + assert_eq!( + g(TraceFieldSelector::ResourceAttribute(vec![ + "service.name".to_string() + ])), + Some("api".to_string()), + ); + assert_eq!( + g(TraceFieldSelector::ScopeAttribute(vec!["lib".to_string()])), + Some("x".to_string()), + ); + } + + #[test] + fn sampling_threshold_writes_tracestate() { + let mut span = val(json!({"name": "x"})); + { + let mut adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.set_field(&TraceFieldSelector::SamplingThreshold, "0"); + } + assert_eq!( + span.get("traceState").and_then(|v| v.as_str()).as_deref(), + Some("ot=th:0"), + ); + } + + #[test] + fn sampling_threshold_merges_existing_tracestate() { + let mut span = val(json!({"traceState": "ot=rv:abc,vendor=1"})); + { + let mut adapter = TraceAdapter { + span: &mut span, + resource: None, + scope: None, + resource_schema_url: None, + scope_schema_url: None, + }; + adapter.set_field(&TraceFieldSelector::SamplingThreshold, "8"); + } + let ts = span + .get("traceState") + .and_then(|v| v.as_str()) + .unwrap() + .into_owned(); + assert!(ts.contains("rv:abc"), "rv preserved: {ts}"); + assert!(ts.contains("th:8"), "th written: {ts}"); + assert!(ts.contains("vendor=1"), "other vendor preserved: {ts}"); + } + + #[test] + fn merge_ot_tracestate_cases() { + assert_eq!(merge_ot_tracestate("", "th:8"), "ot=th:8"); + assert_eq!(merge_ot_tracestate("ot=rv:abc", "th:8"), "ot=rv:abc;th:8"); + assert_eq!(merge_ot_tracestate("foo=bar", "th:8"), "ot=th:8,foo=bar"); + // An existing `th` is overwritten, not duplicated. + assert_eq!(merge_ot_tracestate("ot=th:4", "th:8"), "ot=th:8"); + } +} diff --git a/src/transforms/policy/tests.rs b/src/transforms/policy/tests.rs new file mode 100644 index 0000000000000..f5d851fa10ce8 --- /dev/null +++ b/src/transforms/policy/tests.rs @@ -0,0 +1,2273 @@ +//! End-to-end tests for the `policy` transform driven through Vector's +//! topology layer. +//! +//! These exercise the integration with the `policy-rs` engine using real +//! JSON policy files on disk and the same `create_topology` helper used by +//! every other transform's integration tests. + +use std::io::Write; +use std::path::Path; +use std::time::Duration; + +use serde_json::json; +use tempfile::NamedTempFile; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use vector_lib::config::LogNamespace; +use vector_lib::event::{LogEvent, Metric, MetricKind, MetricValue, TraceEvent}; + +use crate::event::{Event, Value}; +use crate::test_util::components::init_test; +use crate::transforms::test::create_topology; + +use super::config::{PolicyConfig, PolicyMode, PolicyProviderConfig}; +use super::field_mapping::FieldMapping; + +/// Write `body` to a fresh NamedTempFile with the `.json` suffix and return +/// it. The caller must keep the handle alive for the test's duration — +/// dropping it deletes the file out from under the `FileProvider`. +fn write_policies(body: &str) -> NamedTempFile { + let mut file = tempfile::Builder::new() + .prefix("vector-policy-test-") + .suffix(".json") + .tempfile() + .expect("create temp file"); + file.write_all(body.as_bytes()).expect("write policies"); + file.flush().expect("flush policies"); + file +} + +fn policy_config(path: &Path) -> PolicyConfig { + PolicyConfig { + policy_providers: vec![PolicyProviderConfig::file( + "local", + path.to_string_lossy().into_owned(), + )], + mode: PolicyMode::Flat, + field_mapping: FieldMapping::default(), + } +} + +#[allow(dead_code)] +fn policy_config_otel(path: &Path) -> PolicyConfig { + PolicyConfig { + policy_providers: vec![PolicyProviderConfig::file( + "local", + path.to_string_lossy().into_owned(), + )], + mode: PolicyMode::Otel, + field_mapping: FieldMapping::default(), + } +} + +/// Initialize metrics + tracing and build a topology in one step. Every test +/// in this file needs both, so funnel the boilerplate through a single helper. +async fn build_topology( + config: PolicyConfig, +) -> ( + mpsc::Sender, + mpsc::Receiver, + crate::topology::RunningTopology, +) { + init_test(); + let (tx, rx) = mpsc::channel(8); + let (topology, out) = create_topology(ReceiverStream::new(rx), config).await; + (tx, out, topology) +} + +/// Build a representative log event with the fields the default +/// `FieldMapping` expects. +fn log(body: &str) -> Event { + let mut log = LogEvent::default(); + log.insert("message", body); + log.insert("severity_text", "INFO"); + log.into() +} + +/// Receive the next event with a generous timeout — failures here typically +/// mean the transform silently dropped the event rather than yielding it. +async fn recv(out: &mut mpsc::Receiver) -> Option { + tokio::time::timeout(Duration::from_secs(2), out.recv()) + .await + .ok() + .flatten() +} + +/// Confirm the next operation drops, with a timeout so a buggy passthrough +/// can't mask a failure. +async fn assert_no_event(out: &mut mpsc::Receiver) { + let result = tokio::time::timeout(Duration::from_millis(300), out.recv()).await; + assert!( + result.is_err(), + "expected no event but received {:?}", + result.ok().flatten() + ); +} + +const POLICY_DROP_DEBUG: &str = r#"{ + "policies": [ + { + "id": "drop-debug", + "name": "drop-debug", + "log": { + "match": [ + { "log_field": "body", "regex": "debug" } + ], + "keep": "none" + } + } + ] +}"#; + +const POLICY_KEEP_ALL_MATCHING: &str = r#"{ + "policies": [ + { + "id": "keep-errors", + "name": "keep-errors", + "log": { + "match": [ + { "log_field": "body", "regex": "error" } + ], + "keep": "all" + } + } + ] +}"#; + +const POLICY_SAMPLE_ZERO: &str = r#"{ + "policies": [ + { + "id": "drop-sampled", + "name": "drop-sampled", + "log": { + "match": [ + { "log_field": "body", "regex": "drop-me" } + ], + "keep": "0%" + } + } + ] +}"#; + +const POLICY_SAMPLE_HUNDRED: &str = r#"{ + "policies": [ + { + "id": "keep-sampled", + "name": "keep-sampled", + "log": { + "match": [ + { "log_field": "body", "regex": "keep-me" } + ], + "keep": "100%" + } + } + ] +}"#; + +const POLICY_RATE_LIMIT_ONE_PER_SEC: &str = r#"{ + "policies": [ + { + "id": "rate-limit-noisy", + "name": "rate-limit-noisy", + "log": { + "match": [ + { "log_field": "body", "regex": "noisy" } + ], + "keep": "1/s" + } + } + ] +}"#; + +const POLICY_REDACT_PASSWORD: &str = r#"{ + "policies": [ + { + "id": "redact-password", + "name": "redact-password", + "log": { + "match": [ + { "log_field": "body", "regex": "login" } + ], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "password", "replacement": "[REDACTED]" } + ] + } + } + } + ] +}"#; + +const POLICY_REMOVE_DEBUG_TRACE: &str = r#"{ + "policies": [ + { + "id": "remove-debug-trace", + "name": "remove-debug-trace", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "remove": [ + { "log_attribute": "debug_trace" } + ] + } + } + } + ] +}"#; + +const POLICY_RENAME_USER: &str = r#"{ + "policies": [ + { + "id": "rename-user", + "name": "rename-user", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "rename": [ + { "from_log_attribute": "usr", "to": "user_id", "upsert": true } + ] + } + } + } + ] +}"#; + +const POLICY_ADD_PROCESSED_BY: &str = r#"{ + "policies": [ + { + "id": "add-processed-by", + "name": "add-processed-by", + "log": { + "match": [ + { "log_field": "body", "regex": ".+" } + ], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] +}"#; + +// ============================================================================= +// Pass-through and basic filter behaviour. +// ============================================================================= + +#[tokio::test] +async fn empty_policy_file_passes_events_through() { + let policies = write_policies(r#"{ "policies": [] }"#); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("hello world")).await.unwrap(); + let received = recv(&mut out).await.expect("event passed through"); + let log_event = received.into_log(); + assert_eq!( + log_event.get("message").and_then(|v| v.as_str()), + Some("hello world".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn no_matching_policy_passes_event_through() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("regular informational event")).await.unwrap(); + let received = recv(&mut out) + .await + .expect("non-matching event passes through"); + let log_event = received.into_log(); + assert_eq!( + log_event.get("message").and_then(|v| v.as_str()), + Some("regular informational event".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn matched_drop_policy_discards_event() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("debug message")).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn matched_keep_policy_forwards_event() { + let policies = write_policies(POLICY_KEEP_ALL_MATCHING); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + tx.send(log("an error occurred")).await.unwrap(); + let received = recv(&mut out).await.expect("matching keep-all forwards"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("an error occurred".into()) + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Sampling at the extremes (0% drops everything, 100% keeps everything) is +// deterministic and therefore safe to assert. Intermediate percentages are +// inherently probabilistic and intentionally not tested here. +// ============================================================================= + +#[tokio::test] +async fn sample_zero_percent_drops_all_matches() { + let policies = write_policies(POLICY_SAMPLE_ZERO); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + for _ in 0..5 { + tx.send(log("drop-me please")).await.unwrap(); + } + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn sample_hundred_percent_keeps_all_matches() { + let policies = write_policies(POLICY_SAMPLE_HUNDRED); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + for _ in 0..3 { + tx.send(log("keep-me please")).await.unwrap(); + } + for _ in 0..3 { + let received = recv(&mut out).await.expect("100% sample forwards"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("keep-me please".into()) + ); + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Rate-limit policy: the first matching event in the window passes; the +// rest are dropped. Asserting more precise quantities would race with the +// rate-limiter's per-window clock. +// ============================================================================= + +#[tokio::test] +async fn rate_limit_drops_after_first_event() { + let policies = write_policies(POLICY_RATE_LIMIT_ONE_PER_SEC); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // Fire several matching events in quick succession. With 1/s, only the + // first should clear the limiter; subsequent ones are dropped. + for _ in 0..5 { + tx.send(log("noisy chatter")).await.unwrap(); + } + + let first = recv(&mut out).await.expect("first event passes"); + assert_eq!( + first.into_log().get("message").and_then(|v| v.as_str()), + Some("noisy chatter".into()) + ); + + // Nothing else should arrive within the 1-second window. + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Transform actions: redact, remove, rename, add. +// ============================================================================= + +#[tokio::test] +async fn redact_replaces_field_value() { + let policies = write_policies(POLICY_REDACT_PASSWORD); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "user login attempt"); + event.insert("severity_text", "INFO"); + event.insert("attributes.password", "super_secret"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("redact forwards transformed event"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.password").and_then(|v| v.as_str()), + Some("[REDACTED]".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn remove_deletes_field() { + let policies = write_policies(POLICY_REMOVE_DEBUG_TRACE); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.debug_trace", "stack trace here"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("remove forwards transformed event"); + let log = received.into_log(); + assert!(log.get("attributes.debug_trace").is_none()); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rename_moves_field() { + let policies = write_policies(POLICY_RENAME_USER); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.usr", "admin"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("rename forwards transformed event"); + let log = received.into_log(); + assert!(log.get("attributes.usr").is_none()); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("admin".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn add_inserts_field() { + let policies = write_policies(POLICY_ADD_PROCESSED_BY); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("add forwards transformed event"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.processed_by").and_then(|v| v.as_str()), + Some("vector".into()) + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Non-log signals are forwarded untouched. +// ============================================================================= + +#[tokio::test] +async fn metric_event_passes_through_untouched() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + let metric = Event::from(Metric::new( + "test_metric", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + )); + tx.send(metric.clone()).await.unwrap(); + + let received = recv(&mut out).await.expect("metric passes through"); + match received { + Event::Metric(_) => {} // expected + other => panic!("expected metric, got {other:?}"), + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Hot reload: rewriting the policies file changes the engine's behaviour +// without restarting the transform. +// ============================================================================= + +#[tokio::test] +async fn policies_reload_on_file_change() { + use std::io::Seek; + + // Start with a no-op policy file. + let mut policies = write_policies(r#"{ "policies": [] }"#); + let path = policies.path().to_path_buf(); + let config = policy_config(&path); + + let (tx, mut out, topology) = build_topology(config).await; + + // Initially: an event with body "noisy" passes (no policy matches). + tx.send(log("noisy event")).await.unwrap(); + let _ = recv(&mut out).await.expect("initial pass-through"); + + // Rewrite the file with a drop policy. The FileProvider watches the + // parent directory and reloads on change. + policies.as_file_mut().set_len(0).unwrap(); + policies.as_file_mut().rewind().unwrap(); + policies + .write_all( + r#"{ + "policies": [ + { + "id": "drop-noisy", + "name": "drop-noisy", + "log": { + "match": [{ "log_field": "body", "regex": "noisy" }], + "keep": "none" + } + } + ] + }"# + .as_bytes(), + ) + .unwrap(); + policies.as_file_mut().flush().unwrap(); + + // notify-driven reload is asynchronous: poll the transform up to a few + // seconds while the new policy installs. + let mut reloaded = false; + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while std::time::Instant::now() < deadline { + tx.send(log("noisy event")).await.unwrap(); + match tokio::time::timeout(Duration::from_millis(250), out.recv()).await { + Err(_) => { + reloaded = true; + break; + } + Ok(Some(_)) => { + // Old policy still in effect; try again shortly. + tokio::time::sleep(Duration::from_millis(100)).await; + } + Ok(None) => break, + } + } + assert!( + reloaded, + "policy file change did not propagate within 5s; events still pass through" + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Robustness: events that don't carry the body field still flow through. +// ============================================================================= + +#[tokio::test] +async fn event_missing_body_field_passes_through() { + let policies = write_policies(POLICY_DROP_DEBUG); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // No `message` field at all — the body matcher should miss and the + // event should pass through unchanged. + let mut event = LogEvent::default(); + event.insert("attributes.other", "value"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out) + .await + .expect("body-less event passes through"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.other").and_then(|v| v.as_str()), + Some("value".into()) + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn non_string_attribute_satisfies_exists_matcher() { + // Body matchers operate on strings, but `exists` matchers must fire + // for non-string attribute values too — the engine asks + // `Matchable::field_exists`, which our adapter overrides to true for + // any present value type. + let policy = r#"{ + "policies": [ + { + "id": "drop-anything-with-count", + "name": "drop-anything-with-count", + "log": { + "match": [{ "log_attribute": "count", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let config = policy_config(policies.path()); + + let (tx, mut out, topology) = build_topology(config).await; + + // Integer attribute — should still trigger the drop. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.count", 42_i64); + tx.send(event.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // Boolean attribute — should also trigger. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + event.insert("attributes.count", Value::Boolean(true)); + tx.send(event.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // No `count` attribute — should pass through. + let mut event = LogEvent::default(); + event.insert("message", "anything"); + event.insert("severity_text", "INFO"); + tx.send(event.into()).await.unwrap(); + let _ = recv(&mut out).await.expect("missing count -> pass through"); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Matcher-type coverage. Each test confirms that a single matcher variant +// (other than `regex`/`exists`, which already had their own tests above) +// drives a `keep: "none"` policy as expected. +// ============================================================================= + +/// Build a one-policy `keep: "none"` JSON document from a single matcher +/// expressed as a raw JSON snippet (e.g. `r#""exact": "foo""#`). Keeps the +/// matcher tests tiny while still exercising the full deserialize → compile +/// → evaluate chain. +fn drop_policy_with_matcher(matcher_json: &str) -> String { + format!( + r#"{{ + "policies": [ + {{ + "id": "drop", + "name": "drop", + "log": {{ + "match": [{{ "log_field": "body", {matcher_json} }}], + "keep": "none" + }} + }} + ] +}}"# + ) +} + +#[tokio::test] +async fn exact_match_drops_on_full_string_equal() { + let policies = write_policies(&drop_policy_with_matcher(r#""exact": "hello""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("hello")).await.unwrap(); + assert_no_event(&mut out).await; + + // A non-exact match (substring) must NOT drop. + tx.send(log("hello world")).await.unwrap(); + let _ = recv(&mut out).await.expect("non-exact body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn starts_with_match_drops() { + let policies = write_policies(&drop_policy_with_matcher(r#""starts_with": "DEBUG:""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("DEBUG: trace info")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("INFO: app started")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-prefix body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn ends_with_match_drops() { + let policies = write_policies(&drop_policy_with_matcher( + r#""ends_with": "[health-check]""#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("ping [health-check]")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("user request received")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-suffix body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn contains_match_drops() { + let policies = write_policies(&drop_policy_with_matcher(r#""contains": "secret""#)); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("user said the secret word")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("nothing sensitive here")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("non-substring body passes through"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn negate_inverts_match_outcome() { + // Drop everything EXCEPT events whose body equals "keep". + let policies = write_policies(&drop_policy_with_matcher( + r#""exact": "keep", "negate": true"#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("anything")).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("keep")).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("negated match keeps the matching body"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn case_insensitive_match_drops_regardless_of_case() { + let policies = write_policies(&drop_policy_with_matcher( + r#""regex": "error", "case_insensitive": true"#, + )); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + for body in ["ERROR boom", "Error boom", "error boom", "ErRoR boom"] { + tx.send(log(body)).await.unwrap(); + assert_no_event(&mut out).await; + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Match-field namespace coverage: each policy field selector resolves to a +// LogEvent path, and we want one drop-policy test per supported selector to +// catch any regression in the adapter's `path_for` dispatch. +// ============================================================================= + +#[tokio::test] +async fn severity_text_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-debug-severity", + "name": "drop-debug-severity", + "log": { + "match": [{ "log_field": "severity_text", "exact": "DEBUG" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "anything"); + e.insert("severity_text", "DEBUG"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("anything")).await.unwrap(); // severity_text = INFO + let _ = recv(&mut out).await.expect("non-DEBUG passes"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn trace_id_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-trace", + "name": "drop-by-trace", + "log": { + "match": [{ "log_field": "trace_id", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("trace_id", "abc123"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + tx.send(log("x")).await.unwrap(); // no trace_id + let _ = recv(&mut out).await.expect("no trace_id passes"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn span_id_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-span", + "name": "drop-by-span", + "log": { + "match": [{ "log_field": "span_id", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("span_id", "def456"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn event_name_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-event-name", + "name": "drop-by-event-name", + "log": { + "match": [{ "log_field": "event_name", "exact": "noisy.event" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("event_name", "noisy.event"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn resource_schema_url_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-resource-schema", + "name": "drop-by-resource-schema", + "log": { + "match": [{ "log_field": "resource_schema_url", "contains": "v1.0" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert( + "resource.schema_url", + "https://opentelemetry.io/schemas/v1.0", + ); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn scope_schema_url_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-scope-schema", + "name": "drop-by-scope-schema", + "log": { + "match": [{ "log_field": "scope_schema_url", "exists": true }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("scope.schema_url", "https://opentelemetry.io/schemas/foo"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn resource_attribute_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-service", + "name": "drop-by-service", + "log": { + "match": [{ "resource_attribute": "service.name", "exact": "noisy-service" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // The OTel-style "service.name" attribute key contains a literal dot, + // so we insert through a typed path that quotes the segment. + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert(r#"resource.attributes."service.name""#, "noisy-service"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn scope_attribute_matcher_drops() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-scope-attr", + "name": "drop-by-scope-attr", + "log": { + "match": [{ "scope_attribute": "library", "exact": "deprecated-tracer" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("scope.attributes.library", "deprecated-tracer"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Multi-policy and multi-matcher semantics. +// ============================================================================= + +#[tokio::test] +async fn multiple_match_clauses_are_anded() { + // Drops only when BOTH the body matches AND the severity matches. + let policy = r#"{ + "policies": [ + { + "id": "drop-error-from-frontend", + "name": "drop-error-from-frontend", + "log": { + "match": [ + { "log_field": "body", "regex": "error" }, + { "log_field": "severity_text", "exact": "ERROR" } + ], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Both clauses match -> dropped. + let mut e = LogEvent::default(); + e.insert("message", "an error occurred"); + e.insert("severity_text", "ERROR"); + tx.send(e.into()).await.unwrap(); + assert_no_event(&mut out).await; + + // Body matches but severity does not -> passes through. + let mut e = LogEvent::default(); + e.insert("message", "an error occurred"); + e.insert("severity_text", "INFO"); + tx.send(e.into()).await.unwrap(); + let _ = recv(&mut out) + .await + .expect("AND fails on severity mismatch"); + + // Severity matches but body does not -> passes through. + let mut e = LogEvent::default(); + e.insert("message", "all good"); + e.insert("severity_text", "ERROR"); + tx.send(e.into()).await.unwrap(); + let _ = recv(&mut out).await.expect("AND fails on body mismatch"); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn transforms_from_all_matching_policies_apply() { + // Two policies both match the same event. The keep action comes from + // the winning policy (one of them), but per the policy-rs engine, the + // transforms from BOTH matching policies are applied. + let policy = r#"{ + "policies": [ + { + "id": "a-redact-secret", + "name": "a-redact-secret", + "log": { + "match": [{ "log_field": "body", "regex": "audit" }], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "secret", "replacement": "[REDACTED]" } + ] + } + } + }, + { + "id": "b-add-processed-by", + "name": "b-add-processed-by", + "log": { + "match": [{ "log_field": "body", "regex": "audit" }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "audit event"); + e.insert("severity_text", "INFO"); + e.insert("attributes.secret", "hunter2"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.secret").and_then(|v| v.as_str()), + Some("[REDACTED]".into()), + "redact from policy A should have applied", + ); + assert_eq!( + log.get("attributes.processed_by").and_then(|v| v.as_str()), + Some("vector".into()), + "add from policy B should have applied", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn winning_policy_determines_keep_action() { + // Two policies both match. One says "keep: all", the other "keep: none". + // The engine sorts policies by ID alphabetically and selects a winner + // for the keep decision; for this test we just assert the event ends up + // in *some* deterministic state. Because the engine's choice is stable, + // we accept either outcome as long as it's consistent across the batch. + let policy = r#"{ + "policies": [ + { + "id": "aaa-drop", + "name": "aaa-drop", + "log": { + "match": [{ "log_field": "body", "regex": "contested" }], + "keep": "none" + } + }, + { + "id": "zzz-keep", + "name": "zzz-keep", + "log": { + "match": [{ "log_field": "body", "regex": "contested" }], + "keep": "all" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Send a batch — every event must take the same path. If the winner is + // unstable across events the batch would mix drops and passes, which is + // what this test rules out. + for _ in 0..5 { + tx.send(log("contested event")).await.unwrap(); + } + let first = tokio::time::timeout(Duration::from_millis(500), out.recv()).await; + let dropped_consistently = first.is_err(); + let kept_consistently = first.as_ref().map(|r| r.is_some()).unwrap_or(false); + + if kept_consistently { + // First event was kept; the remaining four must also be kept. + for _ in 0..4 { + let _ = recv(&mut out) + .await + .expect("remaining contested events kept"); + } + } else if dropped_consistently { + // First event was dropped; the remaining four must also be dropped. + assert_no_event(&mut out).await; + } else { + panic!("ambiguous outcome: first event returned None during run"); + } + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Transform-op variant coverage. +// ============================================================================= + +#[tokio::test] +async fn redact_with_regex_redacts_substring_only() { + let policy = r#"{ + "policies": [ + { + "id": "redact-cc", + "name": "redact-cc", + "log": { + "match": [{ "log_field": "body", "regex": "card" }], + "keep": "all", + "transform": { + "redact": [ + { + "log_attribute": "card_number", + "replacement": "X", + "regex": "[0-9]" + } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "card event"); + e.insert("severity_text", "INFO"); + e.insert("attributes.card_number", "4242 1111 2222 3333"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.card_number").and_then(|v| v.as_str()), + Some("XXXX XXXX XXXX XXXX".into()), + "regex redact replaces each digit individually", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rename_without_upsert_skips_when_target_exists() { + let policy = r#"{ + "policies": [ + { + "id": "rename-no-upsert", + "name": "rename-no-upsert", + "log": { + "match": [{ "log_field": "body", "regex": "." }], + "keep": "all", + "transform": { + "rename": [ + { "from_log_attribute": "usr", "to": "user_id", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + // Target `user_id` already exists; rename without upsert must leave + // both keys in place (the source is NOT moved). + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("attributes.usr", "admin"); + e.insert("attributes.user_id", "existing"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + let log = received.into_log(); + assert_eq!( + log.get("attributes.usr").and_then(|v| v.as_str()), + Some("admin".into()), + "source key must be preserved when upsert is false and target exists", + ); + assert_eq!( + log.get("attributes.user_id").and_then(|v| v.as_str()), + Some("existing".into()), + "existing target must NOT be overwritten", + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn add_with_upsert_overwrites_existing_value() { + let policy = r#"{ + "policies": [ + { + "id": "add-upsert", + "name": "add-upsert", + "log": { + "match": [{ "log_field": "body", "regex": "." }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": true } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let mut e = LogEvent::default(); + e.insert("message", "x"); + e.insert("severity_text", "INFO"); + e.insert("attributes.processed_by", "something-else"); + tx.send(e.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("event forwarded"); + assert_eq!( + received + .into_log() + .get("attributes.processed_by") + .and_then(|v| v.as_str()), + Some("vector".into()), + "upsert=true overwrites the existing value", + ); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// Misc: trace pass-through + rate-limit-per-minute parse + provider list. +// ============================================================================= + +#[tokio::test] +async fn trace_event_passes_through_untouched() { + use vector_lib::event::TraceEvent; + + let policies = write_policies(POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + let trace = Event::from(TraceEvent::default()); + tx.send(trace).await.unwrap(); + + match recv(&mut out).await { + Some(Event::Trace(_)) => {} + other => panic!("expected trace passthrough, got {other:?}"), + } + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn rate_limit_per_minute_first_event_passes() { + // We can't realistically test the *reset* of a per-minute window in a + // unit test (it would have to wait 60s). This guards the parse path + // and confirms the first matching event is still forwarded. + let policy = r#"{ + "policies": [ + { + "id": "rate-limit-per-minute", + "name": "rate-limit-per-minute", + "log": { + "match": [{ "log_field": "body", "regex": "minutely" }], + "keep": "60/m" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config(policies.path())).await; + + tx.send(log("minutely event")).await.unwrap(); + let _ = recv(&mut out).await.expect("first /m event passes"); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// OTel envelope-mode tests. +// +// These exercise the full envelope iteration path (resourceLogs[]→scopeLogs[] +// →logRecords[]) — per-record filtering, pruning of empty children, and +// dropping the whole event when all records are filtered out. +// ============================================================================= + +/// Build a single OTLP record body wrapped as an AnyValue. +fn otlp_any_string(s: &str) -> serde_json::Value { + json!({ "stringValue": s }) +} + +/// Build a single OTLP attribute entry `{ key, value: { stringValue: ... } }`. +fn otlp_attr(key: &str, value: &str) -> serde_json::Value { + json!({ "key": key, "value": otlp_any_string(value) }) +} + +/// Build an OTLP envelope with a single resourceLogs+scopeLogs entry and +/// the supplied records inside. +fn otlp_envelope(records: Vec) -> Event { + Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { + "attributes": [ + otlp_attr("service.name", "test-service"), + ] + }, + "schemaUrl": "https://opentelemetry.io/schemas/test", + "scopeLogs": [ + { + "scope": { + "name": "test.scope", + "attributes": [ + otlp_attr("library", "test-lib"), + ] + }, + "schemaUrl": "https://opentelemetry.io/schemas/test", + "logRecords": records + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .expect("valid envelope") +} + +/// Build a minimal OTLP log record with the given body string. +fn otlp_record(body: &str) -> serde_json::Value { + json!({ + "severityText": "INFO", + "body": otlp_any_string(body), + "attributes": [] + }) +} + +const OTEL_POLICY_DROP_DEBUG: &str = r#"{ + "policies": [ + { + "id": "drop-debug", + "name": "drop-debug", + "log": { + "match": [{ "log_field": "body", "regex": "debug" }], + "keep": "none" + } + } + ] +}"#; + +#[tokio::test] +async fn otel_mode_passes_non_envelope_event_through() { + // Backwards-compat: if someone configures `mode: otel` but routes a + // non-envelope (flat) event through it, we forward it unchanged + // rather than silently dropping data. + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let mut event = LogEvent::default(); + event.insert("message", "not an envelope"); + tx.send(event.into()).await.unwrap(); + + let received = recv(&mut out).await.expect("flat event passes through"); + assert_eq!( + received.into_log().get("message").and_then(|v| v.as_str()), + Some("not an envelope".into()), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_drops_single_matching_record() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // The single record matches `debug`, so it's filtered out — leaving + // an empty logRecords → empty scopeLogs → empty resourceLogs → the + // whole event is dropped. + let envelope = otlp_envelope(vec![otlp_record("debug message")]); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_filters_some_records_keeps_envelope() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![ + otlp_record("info message"), + otlp_record("debug message"), + otlp_record("warn message"), + otlp_record("another debug here"), + ]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + // Drill into resourceLogs[0].scopeLogs[0].logRecords and verify only + // the two non-debug records remained. + let records = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .expect("records array still present"); + assert_eq!(records.len(), 2); + + let bodies: Vec = records + .iter() + .filter_map(|r| { + r.get("body") + .and_then(|b| b.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()) + }) + .collect(); + assert_eq!(bodies, vec!["info message", "warn message"]); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_prunes_empty_scope_logs() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // Two scopeLogs entries: the first has only matching records, the + // second has a non-matching record. The first entry should be pruned. + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "scopeLogs": [ + { + "scope": { "name": "scope-a" }, + "logRecords": [ + otlp_record("debug 1"), + otlp_record("debug 2"), + ] + }, + { + "scope": { "name": "scope-b" }, + "logRecords": [ + otlp_record("info kept"), + ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let scope_logs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .expect("scopeLogs still present"); + assert_eq!(scope_logs.len(), 1, "empty scope must be pruned"); + let scope_name = scope_logs[0] + .get("scope") + .and_then(|s| s.get("name")) + .and_then(|n| n.as_str()) + .map(|s| s.into_owned()); + assert_eq!(scope_name.as_deref(), Some("scope-b")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_prunes_empty_resource_logs() { + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // Two resourceLogs entries. The first ends up empty after filtering; + // the second still has a record. Expect: only the second remains. + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { "attributes": [ otlp_attr("service.name", "dropped") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("debug a") ] + } + ] + }, + { + "resource": { "attributes": [ otlp_attr("service.name", "kept") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("info b") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let resource_logs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .expect("resourceLogs present"); + assert_eq!( + resource_logs.len(), + 1, + "empty resource entry must be pruned" + ); + + let service = resource_logs[0] + .get("resource") + .and_then(|r| r.get("attributes")) + .and_then(|a| a.as_array()) + .and_then(|a| a.first()) + .and_then(|kv| kv.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(service.as_deref(), Some("kept")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_resource_attribute_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-service", + "name": "drop-by-service", + "log": { + "match": [{ "resource_attribute": "service.name", "exact": "noisy-service" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "resource": { "attributes": [ otlp_attr("service.name", "noisy-service") ] }, + "scopeLogs": [ + { + "scope": { "name": "s" }, + "logRecords": [ otlp_record("hello") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_scope_attribute_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-by-lib", + "name": "drop-by-lib", + "log": { + "match": [{ "scope_attribute": "library", "exact": "deprecated-tracer" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceLogs": [ + { + "scopeLogs": [ + { + "scope": { "attributes": [ otlp_attr("library", "deprecated-tracer") ] }, + "logRecords": [ otlp_record("hello") ] + } + ] + } + ] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_redact_mutates_attribute_array() { + let policy = r#"{ + "policies": [ + { + "id": "redact-secret", + "name": "redact-secret", + "log": { + "match": [{ "log_field": "body", "regex": "login" }], + "keep": "all", + "transform": { + "redact": [ + { "log_attribute": "password", "replacement": "[REDACTED]" } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![json!({ + "severityText": "INFO", + "body": otlp_any_string("user login attempt"), + "attributes": [ + otlp_attr("password", "super-secret"), + otlp_attr("user", "alice"), + ] + })]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let attrs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .and_then(|r| r.first()) + .and_then(|r| r.get("attributes")) + .and_then(|v| v.as_array()) + .expect("attributes array present"); + + let password = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("password")); + let password_val = password + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(password_val.as_deref(), Some("[REDACTED]")); + + // Untouched attribute stays put. + let user_val = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("user")) + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(user_val.as_deref(), Some("alice")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_add_inserts_attribute_into_array() { + let policy = r#"{ + "policies": [ + { + "id": "tag-processed", + "name": "tag-processed", + "log": { + "match": [{ "log_field": "body", "regex": ".+" }], + "keep": "all", + "transform": { + "add": [ + { "log_attribute": "processed_by", "value": "vector", "upsert": false } + ] + } + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![otlp_record("anything")]); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + + let attrs = log + .get("resourceLogs") + .and_then(|v| v.as_array()) + .and_then(|rl| rl.first()) + .and_then(|rl| rl.get("scopeLogs")) + .and_then(|v| v.as_array()) + .and_then(|sl| sl.first()) + .and_then(|sl| sl.get("logRecords")) + .and_then(|v| v.as_array()) + .and_then(|r| r.first()) + .and_then(|r| r.get("attributes")) + .and_then(|v| v.as_array()) + .expect("attributes array present"); + + let processed_by = attrs + .iter() + .find(|a| a.get("key").and_then(|k| k.as_str()).as_deref() == Some("processed_by")) + .and_then(|a| a.get("value")) + .and_then(|v| v.get("stringValue")) + .and_then(|s| s.as_str()) + .map(|s| s.into_owned()); + assert_eq!(processed_by.as_deref(), Some("vector")); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_severity_text_matcher_drops_record() { + let policy = r#"{ + "policies": [ + { + "id": "drop-debug-severity", + "name": "drop-debug-severity", + "log": { + "match": [{ "log_field": "severity_text", "exact": "DEBUG" }], + "keep": "none" + } + } + ] + }"#; + let policies = write_policies(policy); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_envelope(vec![json!({ + "severityText": "DEBUG", + "body": otlp_any_string("anything"), + "attributes": [] + })]); + tx.send(envelope).await.unwrap(); + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_metric_event_passes_through_untouched() { + // Same guarantee as flat mode: non-log signals are forwarded as-is. + let policies = write_policies(OTEL_POLICY_DROP_DEBUG); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let metric = Event::from(Metric::new( + "test", + MetricKind::Incremental, + MetricValue::Counter { value: 1.0 }, + )); + tx.send(metric).await.unwrap(); + let received = recv(&mut out).await.expect("metric passes through"); + assert!(matches!(received, Event::Metric(_))); + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// OTel mode: OTLP metrics envelopes (`resourceMetrics`). +// ============================================================================= + +const OTEL_POLICY_DROP_METRIC_BY_NAME: &str = r#"{ + "policies": [ + { + "id": "drop-system-load", + "name": "drop-system-load", + "metric": { + "match": [ { "metric_field": "name", "regex": "^system\\.load.*$" } ], + "keep": false + } + } + ] +}"#; + +#[tokio::test] +async fn otel_mode_metrics_envelope_drops_and_prunes() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // scope-a holds only a system.load metric (dropped → scope pruned); + // scope-b holds system.load (dropped) + http.requests (kept). This + // exercises the two-phase keep/prune index bookkeeping across scopes. + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "resource": { "attributes": [] }, + "scopeMetrics": [ + { + "scope": { "name": "scope-a" }, + "metrics": [ + { "name": "system.load.1", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ] + }, + { + "scope": { "name": "scope-b" }, + "metrics": [ + { "name": "system.load.5", "gauge": { "dataPoints": [{ "attributes": [] }] } }, + { "name": "http.requests", "sum": { "dataPoints": [{ "attributes": [] }], "aggregationTemporality": "AGGREGATION_TEMPORALITY_DELTA" } } + ] + } + ] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("metrics envelope forwarded"); + let log = received.into_log(); + + let scope_metrics = log + .get("resourceMetrics") + .and_then(|v| v.as_array()) + .and_then(|rm| rm.first()) + .and_then(|rm| rm.get("scopeMetrics")) + .and_then(|v| v.as_array()) + .expect("scopeMetrics present"); + assert_eq!(scope_metrics.len(), 1, "empty scope-a must be pruned"); + assert_eq!( + scope_metrics[0] + .get("scope") + .and_then(|s| s.get("name")) + .and_then(|n| n.as_str()) + .as_deref(), + Some("scope-b"), + ); + let metrics = scope_metrics[0] + .get("metrics") + .and_then(|v| v.as_array()) + .expect("metrics present"); + assert_eq!(metrics.len(), 1, "only the non-matching metric survives"); + assert_eq!( + metrics[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("http.requests"), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_metrics_envelope_all_dropped_drops_event() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "scopeMetrics": [{ + "metrics": [ + { "name": "system.load.1", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ] + }] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + // The only metric is dropped, so the whole envelope is pruned away. + assert_no_event(&mut out).await; + + drop(tx); + topology.stop().await; +} + +// ============================================================================= +// OTel mode: OTLP traces envelopes (`resourceSpans`). +// ============================================================================= + +const OTEL_POLICY_DROP_TRACE_BY_NAME: &str = r#"{ + "policies": [ + { + "id": "drop-basic", + "name": "drop-basic", + "trace": { + "match": [ { "trace_field": "TRACE_FIELD_NAME", "exact": "basic" } ], + "keep": { "percentage": 0.0 } + } + } + ] +}"#; + +const OTEL_POLICY_KEEP_ERROR_SPANS: &str = r#"{ + "policies": [ + { + "id": "keep-error", + "name": "keep-error", + "trace": { + "match": [ { "span_status": "SPAN_STATUS_CODE_ERROR", "exists": true } ], + "keep": { "percentage": 100.0 } + } + } + ] +}"#; + +fn otlp_trace(value: serde_json::Value) -> Event { + Event::Trace(TraceEvent::from(Value::from(value))) +} + +#[tokio::test] +async fn otel_mode_traces_envelope_drops_and_prunes() { + let policies = write_policies(OTEL_POLICY_DROP_TRACE_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "resource": { "attributes": [] }, + "scopeSpans": [{ + "scope": { "name": "s" }, + "spans": [ + { "name": "basic", "spanId": "a", "traceId": "t" }, + { "name": "keep-me", "spanId": "b", "traceId": "t" } + ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("traces envelope forwarded"); + let spans = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .and_then(|rs| rs.get("scopeSpans")) + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .expect("spans present"); + assert_eq!(spans.len(), 1, "the matching span is dropped"); + assert_eq!( + spans[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("keep-me"), + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_traces_sampling_writes_tracestate() { + let policies = write_policies(OTEL_POLICY_KEEP_ERROR_SPANS); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "scopeSpans": [{ + "spans": [ + { "name": "checkout", "spanId": "a", "traceId": "t", "status": { "code": "STATUS_CODE_ERROR" } } + ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("traces envelope forwarded"); + let span = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .and_then(|rs| rs.get("scopeSpans")) + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .and_then(|s| s.first()) + .cloned() + .expect("span kept"); + // 100% sampling writes the OTel threshold `th=0` into the tracestate. + assert_eq!( + span.get("traceState").and_then(|v| v.as_str()).as_deref(), + Some("ot=th:0"), + ); + + drop(tx); + topology.stop().await; +} + +// --- Envelope edge cases (guard the two-phase / lift hardening) ------------- + +#[tokio::test] +async fn otel_mode_metrics_scope_without_metrics_key_passes_through() { + let policies = write_policies(OTEL_POLICY_DROP_METRIC_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // One scope has no `metrics` array at all (empty decision row in the + // two-phase pass); it must survive untouched rather than panic or prune. + let envelope = Event::from_json_value( + json!({ + "resourceMetrics": [{ + "scopeMetrics": [ + { "scope": { "name": "no-metrics" } }, + { "scope": { "name": "has-metrics" }, "metrics": [ + { "name": "http.requests", "gauge": { "dataPoints": [{ "attributes": [] }] } } + ]} + ] + }] + }), + LogNamespace::Legacy, + ) + .unwrap(); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("envelope forwarded"); + let log = received.into_log(); + let scope_metrics = log + .get("resourceMetrics") + .and_then(|v| v.as_array()) + .and_then(|rm| rm.first()) + .and_then(|rm| rm.get("scopeMetrics")) + .and_then(|v| v.as_array()) + .expect("scopeMetrics present"); + assert_eq!( + scope_metrics.len(), + 2, + "the metrics-less scope must survive" + ); + + drop(tx); + topology.stop().await; +} + +#[tokio::test] +async fn otel_mode_traces_without_resource_or_scope() { + let policies = write_policies(OTEL_POLICY_DROP_TRACE_BY_NAME); + let (tx, mut out, topology) = build_topology(policy_config_otel(policies.path())).await; + + // No `resource` / `scope` objects: `lift_child` returns `None`. A kept span + // must survive, and the absent resource must not be synthesized on output. + let envelope = otlp_trace(json!({ + "resourceSpans": [{ + "scopeSpans": [{ + "spans": [ { "name": "keep-me", "spanId": "a", "traceId": "t" } ] + }] + }] + })); + tx.send(envelope).await.unwrap(); + + let received = recv(&mut out).await.expect("trace forwarded"); + let first_rs = received + .as_trace() + .get("resourceSpans") + .and_then(|v| v.as_array()) + .and_then(|rs| rs.first()) + .cloned() + .expect("resourceSpans present"); + assert!( + first_rs.get("resource").is_none(), + "absent resource must not be synthesized", + ); + let spans = first_rs + .get("scopeSpans") + .and_then(|v| v.as_array()) + .and_then(|ss| ss.first()) + .and_then(|ss| ss.get("spans")) + .and_then(|v| v.as_array()) + .expect("spans present"); + assert_eq!(spans.len(), 1); + assert_eq!( + spans[0].get("name").and_then(|n| n.as_str()).as_deref(), + Some("keep-me"), + ); + + drop(tx); + topology.stop().await; +} diff --git a/src/transforms/policy/transform.rs b/src/transforms/policy/transform.rs new file mode 100644 index 0000000000000..a245d98965f74 --- /dev/null +++ b/src/transforms/policy/transform.rs @@ -0,0 +1,153 @@ +//! The async task transform that drives the `policy-rs` engine. + +use std::pin::Pin; +use std::sync::Arc; + +use async_stream::stream; +use futures::{Stream, StreamExt}; +use policy_rs::{EvaluateResult, PolicyEngine, PolicyRegistry, PolicySnapshot}; +use vector_lib::transform::TaskTransform; + +use crate::event::{Event, LogEvent}; + +use super::adapter::VectorLogAdapter; +use super::config::PolicyMode; +use super::field_mapping::FieldMapping; +use super::internal_events::{DropReason, emit_dropped}; +use super::otlp_log_adapter::evaluate_logs_envelope; +use super::otlp_metric_adapter::evaluate_metrics_envelope; +use super::otlp_trace_adapter::evaluate_traces_envelope; + +/// Per-task state for the `policy` transform. +/// +/// `PolicyRegistry` and `PolicyEngine` are wrapped in `Arc` so the +/// `TaskTransform` machinery can move the struct freely while preserving +/// rate-limit and stats state across reconfigurations. +#[derive(Clone)] +pub struct Policy { + registry: Arc, + engine: Arc, + mapping: Arc, + mode: PolicyMode, +} + +impl Policy { + pub const fn new( + registry: Arc, + engine: Arc, + mapping: Arc, + mode: PolicyMode, + ) -> Self { + Self { + registry, + engine, + mapping, + mode, + } + } +} + +impl TaskTransform for Policy { + fn transform( + self: Box, + mut input_rx: Pin + Send>>, + ) -> Pin + Send>> + where + Self: 'static, + { + let registry = Arc::clone(&self.registry); + let engine = Arc::clone(&self.engine); + let mapping = Arc::clone(&self.mapping); + let mode = self.mode; + + Box::pin(stream! { + while let Some(event) = input_rx.next().await { + // Pull a fresh snapshot every event so live-reloaded policies + // take effect on the next pass. Snapshots are cheap (Arc clone + // of an immutable inner). + let snapshot = registry.snapshot(); + match event { + Event::Log(log) => { + let outcome = match mode { + PolicyMode::Flat => { + evaluate_flat(&engine, &snapshot, &mapping, log).await + } + // In OTel mode a `Log` event is either a logs + // envelope (`resourceLogs`) or a metrics envelope + // (`resourceMetrics`) — the `opentelemetry` source + // decodes both into `Log` events. + PolicyMode::Otel if log.contains("resourceMetrics") => { + evaluate_metrics_envelope(&engine, &snapshot, log).await + } + PolicyMode::Otel => { + evaluate_logs_envelope(&engine, &snapshot, log).await + } + }; + if let Some(forwarded) = outcome { + yield Event::Log(forwarded); + } + } + Event::Trace(mut trace) => { + // OTLP traces arrive as `Trace` events + // (`resourceSpans`). Flat mode has no trace mapping, so + // only OTel mode evaluates them; otherwise pass through. + let keep = match mode { + PolicyMode::Otel => { + evaluate_traces_envelope(&engine, &snapshot, &mut trace).await + } + PolicyMode::Flat => true, + }; + if keep { + yield Event::Trace(trace); + } + } + // Native Vector metrics (not OTLP envelopes) pass through. + other => { + yield other; + } + } + } + }) + } +} + +/// Flat-mode evaluation: one Vector event = one log record. Returns +/// `Some(log)` to forward, `None` to drop. +async fn evaluate_flat( + engine: &PolicyEngine, + snapshot: &PolicySnapshot, + mapping: &FieldMapping, + mut log: LogEvent, +) -> Option { + let result = { + let mut adapter = VectorLogAdapter::new(&mut log, mapping); + engine.evaluate_and_transform(snapshot, &mut adapter).await + }; + match result { + Ok(EvaluateResult::NoMatch) + | Ok(EvaluateResult::Keep { .. }) + | Ok(EvaluateResult::Sample { keep: true, .. }) + | Ok(EvaluateResult::RateLimit { allowed: true, .. }) => Some(log), + Ok(EvaluateResult::Drop { .. }) => { + emit_dropped(DropReason::PolicyDrop, 1); + None + } + Ok(EvaluateResult::Sample { keep: false, .. }) => { + emit_dropped(DropReason::SampleRejected, 1); + None + } + Ok(EvaluateResult::RateLimit { allowed: false, .. }) => { + emit_dropped(DropReason::RateLimited, 1); + None + } + Err(error) => { + // Fail open: an evaluation error shouldn't silently drop + // telemetry. Log and pass the event through untouched. + error!( + message = "Policy evaluation failed; event passed through unchanged.", + %error, + ); + Some(log) + } + } +}